news_processer.py

import configparser
import codecs
import json
from normalization import normalization
import datetime, dateutil
from utils import make_links, LastNamesExtractor, check_keywords_occur
from sklearn.feature_extraction.text import TfidfVectorizer
import networkx
from operator import itemgetter
import numpy as np


class NewsProcesser:


    def __init__(self):
        self.extractor = LastNamesExtractor()
        #config.read('config.ini')

        #host_mongo = config['MongoParams']['host']


    def cluster_news(self, data, valid_tags=None, keywords=None, language='ru', min_words_num=10, thresh_dist_links=0.55, thresh_dist_same=0.25, thresh_hours=24, min_cluster_size=5, extra_stop_words=None):

        """
        data - массив json, содержащих поля:
            text - текст новости,
            title - название новости,
            tags - массив тегов новости, например ['politics', 'covid'].
                   При итерации data, будут учитываться items, имеющих тег из поля valid_tags или содержащих ключевые слова keywords
            ts - unix timestamp новости,
            source - источник новсти (url, название канала, группы vk итд......)

        valid_tags - Валидные теги новости. Если None, то учитывем все новости
        keywords - Ключевые слова для фильтрации текста
        language - Язык текста. Может быть или 'ru' или 'en'
        min_words_num - Минимальное количество слов (не включая стоп слова), когда текст считается валидным
        thresh_dist_links - Порог связности новостей
        thresh_dist_same - Порог схожести новостей
        thresh_hours - Время в часах, когда считать, что новости c расстоянием меньшим, чем thresh_dist_same, одинаковы

        """


        if keywords is None:
            keywords = set()
        else:
            keywords = set(keywords)


        n_valid_text = 0
        normalized_corpus = []
        valid_ids = []

        for n_item, item in enumerate(data):
            #siteuper tags disable
            #tags = item['tags']
            #if tags is None:
               #tags = []
            text = item['text']


            pretty_text, norm_text = normalization.normalize_text(text=text,
                                                                  exclude_other_languages=True,
                                                                  language=language, extra_stop_words=extra_stop_words)

            norm_text = norm_text.replace(' EOS', '')
            norm_words = norm_text.split()
            if len(norm_words) < min_words_num:
                continue


            valid_condition = (valid_tags is None or
                               len(set(valid_tags).intersection(tags)) > 0 or
                               len(set(norm_words).intersection(keywords)) > 0)

            if not valid_condition:
                continue


            norm_text = ' '.join([word for word in norm_words if word not in keywords])
            normalized_corpus.append(norm_text)
            valid_ids.append(n_item)
            n_valid_text+=1


        dt_vector = [datetime.datetime.utcfromtimestamp(data[valid_id]['ts']) for valid_id in valid_ids]


        vectorizer = TfidfVectorizer(ngram_range=(1, 2), min_df=4, max_df=0.5)
        feature_matrix = vectorizer.fit_transform(normalized_corpus).astype(float)
        d_links, d_same, d_power = make_links(feature_matrix,
                                              dt_vector,
                                              thresh_dist_links=thresh_dist_links,
                                              thresh_dist_same=thresh_dist_same,
                                              thresh_hours=thresh_hours)

        G = networkx.Graph(d_links)
        #components = [(connected_component, len(connected_component)) for connected_component in networkx.connected_components(G)]
        #components = sorted(components, key=i)

        clusters = []
        for connected_component in networkx.connected_components(G):
            if len(connected_component) < min_cluster_size:
                continue
            connected_component_l = list(connected_component)
            sored_args = np.argsort([dt_vector[idx] for idx in connected_component_l])
            d_out = {}
            d_out['size'] = len(connected_component)
            output = []
            for arg in sored_args:
                d_out = {}
                main_id = connected_component_l[arg]
                d_out['sources'] = [data[valid_ids[main_id]]['source']] + [data[valid_ids[same_id]]['source'] for same_id in d_same[main_id]]
                #d_out['urls'] = [data[valid_ids[main_id]]['url']] + [data[valid_ids[same_id]]['url'] for
                #                                                           same_id in d_same[main_id]]
                d_out['t'] = dt_vector[main_id].strftime('%Y-%m-%d %H:%M:%S')
                d_out['text'] = data[valid_ids[main_id]]['text']
                output.append(d_out)
            clusters.append(output)
        return clusters

    def extract_names(self, data, valid_tags=None, keywords=None):


        """
        data - массив json, который должен содержать содержащих поля:
            text - текст новости
            tags - массив тегов новости, например ['politics', 'covid'].
            При итерации data, будут учитываться items, имеющих тег из поля valid_tags или содержащих ключевые слова keywords
        """

        output = []
        if keywords is not None:
            keywords = set(keywords)

        if valid_tags is not None:
            valid_tags = set(valid_tags)
        else:
            valid_tags = set()

        for n_item, item in enumerate(data):


            valid_text = False

            tags = item['tags']
            if tags is None:
                tags = []

            if valid_tags and valid_tags.intersection(tags):
                valid_text = True


            text = item['text']

            if (not valid_text) and keywords:
                pretty_text, norm_text = normalization.normalize_text(text=text,
                                                                      exclude_other_languages=True,
                                                                      language='ru')
                norm_text = norm_text.replace(' EOS', '')
                norm_words = norm_text.split()
                if len(set(norm_words).intersection(keywords)) > 0:
                    valid_text = True

            if not valid_text:
                continue

            names = self.extractor.extract_names(item['text'])
            names_item = [(n_item, name) for name in names]
            output.extend(names_item)
        return output



    def extract_filter(self, data, keywords_array, filters=None, keywords_distance=4):




        output = []


        for n_item, item in enumerate(data):

            text = item['text']


            pretty_text, norm_text = normalization.normalize_text(text=text,
                                                                  exclude_other_languages=True,
                                                                  use_abbr=True,
                                                                  language='ru')

            norm_text = norm_text.replace(' EOS', '')
            norm_words = norm_text.split()
            if filters and len(set(norm_words).intersection(filters)) == 0:
                continue

            for keywords in keywords_array:

                if len(set(norm_words).intersection(keywords)) == len(keywords):

                    if len(keywords) == 1:
                        break


                    M = np.zeros((len(keywords), len(norm_words)))

                    for i, keyword in enumerate(keywords):
                       for j, norm_word in enumerate(norm_words):
                           if keyword == norm_word:
                               M[i,j] = 1
                   #M_sum = np.sum(M, axis=0).astype(bool).astype(int)

                    valid_cond = False
                    for i in range(0, M.shape[1] - keywords_distance):
                        valid_cond = np.prod(M[:, i:i+keywords_distance].sum(axis=1))
                        if valid_cond:
                            break
                    if valid_cond == True:
                        break
            else:
                continue

            print(item)
            item['t'] = datetime.datetime.utcfromtimestamp(item['ts']).strftime('%Y-%m-%d %H:%M:%S')
            output.append(item)

        return output






    def extract_true_comments(self, data, keywords_array, filters=None, keywords_distance=4):



        log_f = 'comments_log.txt'

        f = codecs.open(log_f, 'w', encoding='utf-8')

        f.write(str(len(data)))
        f.write('\n')
        f.flush()

        output_comments = []
        output_messages = []


        start = datetime.datetime.now()


        for n_item, item in enumerate(data):

            if n_item % 1000 == 0:
                s = 'items {}: {} seconds'.format(n_item, (datetime.datetime.now()-start).seconds)
                f.write(s)
                f.write('\n')
                f.flush()




            comments = item.get('comments')
            if comments is None:
                continue
            d_out = {}

            d_out['text'] = item['text']
            d_out["t"] = datetime.datetime.utcfromtimestamp(item["ts"]).strftime('%Y-%m-%d %H:%M:%S')
            d_out["source"] = item['source']
            d_out['n_likes'] = item['likes_count']
            d_out['title'] = item['title']
            d_out['n_views'] = item['n_views']
            d_out['n_comments'] = item['comments_count']

            _, norm_text = normalization.normalize_text(item['text'],
                                                        exclude_other_languages=True,
                                                        use_abbr=True,
                                                        language='ru')

            norm_text = norm_text.replace(' EOS', '')
            norm_words = norm_text.split()
            for keywords in keywords_array:
                res = check_keywords_occur(keywords=keywords, norm_words=norm_words,
                                           keywords_distance=keywords_distance)
                if res:
                    output_messages.append(d_out.copy())
                    break



            valid_comments = []

            for comment in comments:
                comment_text = comment.get('text')
                if comment_text is None:
                    continue

                _, norm_text = normalization.normalize_text(text=comment_text,
                                                                      exclude_other_languages=True,
                                                                      use_abbr=True,
                                                                      language='ru')

                norm_text = norm_text.replace(' EOS', '')
                norm_words = norm_text.split()
                for keywords in keywords_array:
                    res = check_keywords_occur(keywords=keywords, norm_words=norm_words, keywords_distance=keywords_distance)
                    if res:
                        break
                else:
                    continue
                valid_comments.append(comment_text)

            if valid_comments:
                d_out['valid_comments'] = valid_comments
                output_comments.append(d_out)

        return output_messages, output_comments