[ python] 뉴스 분석기 feat.네이버 개발자센터 , 형태소 분석

News.py

News.py

0.00MB

 

 

#News.py
class News:
    def __init__(self,title,link,description,pubdate):
        self.title = title
        self.link = link
        self.description = description
        self.pubdate = pubdate
    @staticmethod
    def MakeNews(jnews):
        title = jnews['title']
        link = jnews['link']
        description = jnews['description']
        pubdate = jnews['pubDate']
        return News(title,link,description,pubdate)

 

Morpheme.py

Morpheme.py

0.00MB

 

 

 

#Morpheme.py  - 형태소 클래스
class Morpheme:
    def __init__(self,word):
        self.word = word #단어 
        self.ref = 1           #참조 개수
    def Merge(self,other): # 병합하기
        if self.IsEqual(other):
            self.ref = self.ref + other.ref
    def IsEqual(self,other): #같은 단어를 갖는 형태소인지 판별
        return self.word ==  other.word

MorphemeParser.py

MorphemeParser.py

0.00MB

 

 

#MorphemeParser.py - 형태소 분석기
from Morpheme import Morpheme
class MorphemeParser:
    @staticmethod
    def Parse(src):
        morphes = list() 
        #원본 문자열에 특수 기호를 제거 및 공백 기준으로 분리
        src = MorphemeParser.RemoveTag(src)
        src = MorphemeParser.RemoveNonAlpha(src)
        msrc = src.split(' ')
        #각 단어를 형태소 컬렉션에 추가
        for elem in msrc:
            if str.isalpha(elem):
                morphes.append(Morpheme(elem))
        #중복 형태소를 합치는 공정
        morphes = MorphemeParser.Merge(morphes)
        return morphes

    @staticmethod
    def Merge(morphes):
        remoes = list()#병합한 형태소를 보관할 컬렉션
        for morph in morphes:#원본 컬렉션에 있는 각각의 형태소를
            rcnt = len(remoes)#병합한 컬렉션에 형태소 개수 구하기
            flag = False #morph와 같은 단어가 remoes에 없다고 가정
            #morph가 remoes컬렉션에 있다면 병합
            for index in range(0,rcnt):
                if remoes[index].word == morph.word:
                    remoes[index].Merge(morph)
                    flag = True#병합하였음을 마킹
                    break            
            if flag == False:#morph와 같은 단어는 remoes에 없음
                remoes.append(morph)
        return remoes
    @staticmethod
    def RemoveNonAlpha(src):
        dest=""
        for elem in src:
            if str.isalpha(elem) or str.isspace(elem):
                dest += elem
        return dest
    @staticmethod
    def FindTag(src):
        s = src.index('<')
        e = src.index('>')
        return s,e
    @staticmethod
    def RemoveTag(src):
        try:
            while True:
                s,e = MorphemeParser.FindTag(src)
                src = src[0:s]+src[e+1:] #태그를 제거하는 구문
        except:# 더 이상 태그가 없음
            return src

NewsSearcher.py

NewsSearcher.py

0.00MB

 

 

#NewsSearcher.py - 뉴스 검색기
from MorphemeParser import MorphemeParser
from Morpheme import Morpheme
import urllib.request
import json
class NewsSearcher:
    def __init__(self):
        self.client_id ="네이버에서 발급한 클라이언트 id"
        self.client_secret="네이버에서 발급한 클라이언트 secret"
        self.url = "https://openapi.naver.com/v1/search/news.json"
    def SetQuery(self,query):
        query = urllib.parse.quote(query)
        self.qp = "query="+query
    def Request(self,start,display):
        sp = "start="+str(start)
        dp = "display="+str(display)
        query_str = self.url+"?"+self.qp+"&"+sp+"&"+dp
        request = urllib.request.Request(query_str)
        request.add_header("X-Naver-Client-Id",self.client_id)
        request.add_header("X-Naver-Client-Secret",self.client_secret)
        try:
            response = urllib.request.urlopen(request)
        except: #예외 발생하였을 때
            return list(),0
        if response.getcode()!=200:#실패일 때
            return list(),0
        content = response.read()
        content = content.decode('utf-8')
        jdata = json.loads(content)
        total = int(jdata['total'])
        return jdata['items'],total
    def RequestAll(self):
        start = 1
        display = 100
        redatas = list()
        datas, total = self.Request(start,display)
        redatas.extend(datas)
        start = start + display
        while start<total and start<1000:
            datas, total = self.Request(start,display)
            redatas.extend(datas)
            start = start + display
        return redatas

NewsAnaylizer.py

NewsAnaylizer.py

0.00MB

 

 

#NewsAnaylizer.py - 뉴스 분석기
from NewsSearcher import NewsSearcher
from MorphemeParser import MorphemeParser
from Morpheme import Morpheme
from News import News
class NewsAnaylizer:
    def __init__(self):
        self.ns = NewsSearcher()
    def Analize(self, query):
        redata = list()
        self.ns.SetQuery(query)
        news_col = self.ns.RequestAll()
        for jnews in news_col:
            news = News.MakeNews(jnews)
            morphes1 = MorphemeParser.Parse(news.title)
            morphes2 = MorphemeParser.Parse(news.description)
            redata.append([news,morphes1,morphes2])
        return redata

Main.py

from NewsAnaylizer import NewsAnaylizer
from MorphemeParser import MorphemeParser
import matplotlib.pyplot as plt
import matplotlib
na = NewsAnaylizer()
q = input('질의:')
moes = list()
adatas = na.Analize(q)
for adata in adatas:
    news,m1,m2 = adata
    moes.extend(m1)
    moes.extend(m2)
    print(news.title)
    print(news.description)
    print("=======")
input("엔터 키를 누르세요.")
moes = MorphemeParser.Merge(moes)

reflist = list()
for mo in moes:
    print("{0},{1}".format(mo.word,mo.ref))
    reflist.append(mo.ref)
plt.plot(reflist)
plt.show()