-
Notifications
You must be signed in to change notification settings - Fork 1
/
extract_entries.py
42 lines (37 loc) · 1.04 KB
/
extract_entries.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import re
import warc
from selectolax.parser import HTMLParser
def get_text_from_warc(file):
f = warc.open(file)
output = []
with open('entries_single_page.txt', 'w', encoding="utf-8") as outfile:
for record in f:
if record.header.get("WARC-Trec-ID"):
outfile.write('WARC-Trec-ID: ' + record.header.get("WARC-Trec-ID") + '\n')
if record.url:
payload = record.payload.read()
header, html = payload.split(b'\r\n\r\n', maxsplit=1)
html = html.strip()
if len(html) > 0:
text = get_text_from_html(html)
if text:
text = cleanup(text)
for entry in text.replace('\n', ' ').split():
outfile.write(entry + '\n')
break
return output
def cleanup(s):
s = re.sub(' +', ' ', s)
s = re.sub(r'[^a-zA-Z0-9]+', ' ', s)
return s
def get_text_from_html(html):
tree = HTMLParser(html)
if tree.body is None:
return None
for tag in tree.css('script'):
tag.decompose()
for tag in tree.css('style'):
tag.decompose()
text = tree.body.text(separator='\n')
return text
get_text_from_warc('sample.warc.gz')