Tagger/tagger.py

88 lines
2.3 KiB
Python
Raw Normal View History

2022-05-05 16:01:10 +00:00
import re
2022-05-05 17:30:56 +00:00
from dataclasses import dataclass
from html.parser import HTMLParser
from pathlib import Path
from exclude import EXCLUDED_WORDS
upper_check = re.compile(r'[A-Z]')
2022-05-05 16:01:10 +00:00
2022-05-05 17:30:56 +00:00
@dataclass
class Tag:
name: str
score: int
class FileScanner(HTMLParser):
def __init__(self, file: Path):
super().__init__()
self.file = file
self.texte = []
def scan_file(self):
content = read_file(self.file)
self.feed(content)
words_with_usage = {}
words = []
for text in self.texte:
words += re.split(r'[ /\-_#\n]', text)
print(f'\nFile {self.file.parent.name} contains {len(words)} words')
title_words = set(self.file.parent.name.split('-'))
for word in words:
tag_name = word.strip(".,:;!\"'<>()")
word = tag_name.lower()
if not word:
continue
score = 10
if word in EXCLUDED_WORDS:
score = 0
if word in title_words:
score *= 4
if len(word) <= 3:
score //= 2
upper_letters_count = len(upper_check.findall(tag_name))
score *= upper_letters_count
if word not in words_with_usage:
words_with_usage[word] = Tag(name=tag_name, score=score)
else:
words_with_usage[word].score += score
sorted_list = sorted(words_with_usage.values(), key=lambda tag: tag.score, reverse=False)
display_result(sorted_list)
def handle_starttag(self, tag, attrs):
if tag != "a":
return
for attr_name, attr_value in attrs:
if attr_name == "href":
self.texte.append(attr_value)
break
2022-05-05 16:01:10 +00:00
2022-05-05 17:30:56 +00:00
def handle_data(self, data):
self.texte.append(data)
2022-05-05 16:01:10 +00:00
def display_result(result):
2022-05-05 17:30:56 +00:00
for tag in result:
if tag.score <= 10:
continue
print(f"Score: {tag.score:>3} Word: {tag.name}")
2022-05-05 16:01:10 +00:00
def read_file(file: Path) -> str:
with open(file, 'r') as file:
return file.read()
def main(source=Path('data')):
2022-05-05 17:30:56 +00:00
for index, file in enumerate(source.glob('**/index.txt')):
scanner = FileScanner(file)
scanner.scan_file()
# if index == 3:
# break
2022-05-05 16:01:10 +00:00
if __name__ == '__main__':
main()