-
Notifications
You must be signed in to change notification settings - Fork 0
/
ethnoExtraction.py
112 lines (89 loc) · 2.42 KB
/
ethnoExtraction.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
from tulip import *
import json
from bs4 import BeautifulSoup
stfGroups = set(['2607', '2606', '2613', '2762', '2877', '2873', '2837', '2795', '709STF'])
stfPath = "/work/EdgeRyders2/stf-network-data-"
stfEthnoPost = "ethno-posts.json"
nodesKey = "nodes"
nodeKey = "node"
ethnoNIDKey = u'Nid'
ethnoBodyKey = u'Body'
nIDToEthnoPost = {}
categoryToTag = {}
stfEthnoComments1 = "ethno-comments.json"
stfEthnoComments2 = "ethno-comments2.json"
stfEthnoComments3 = "ethno-comments3.json"
nIDToEthnoComment = {}
ethnoCIDKey = u'ID'
ethnoCommentKey = u'Comment'
def fillMapArray(entry, val, hashMap):
if entry not in hashMap:
hashMap[entry] = []
hashMap[entry].append(val)
def loadOneEthnoPost(dpN):
dpN = dpN[nodeKey]
post = dpN[ethnoBodyKey]
nID = dpN[ethnoNIDKey]
ethno = chunkPost(post)
nIDToEthnoPost[nID] = post
def loadOneEthnoComment(dpN):
dpN = dpN[nodeKey]
post = dpN[ethnoCommentKey]
nID = dpN[ethnoCIDKey]
ethno = chunkPost(post)
nIDToEthnoComment[nID] = post
def loadEthnoComments():
f = open(stfPath+stfEthnoComments1)
dpNodes = json.load(f)
for dpN in dpNodes[nodesKey]:
loadOneEthnoComment(dpN)
f.close()
f = open(stfPath+stfEthnoComments2)
dpNodes = json.load(f)
for dpN in dpNodes[nodesKey]:
loadOneEthnoComment(dpN)
f.close()
f = open(stfPath+stfEthnoComments3)
dpNodes = json.load(f)
for dpN in dpNodes[nodesKey]:
loadOneEthnoComment(dpN)
f.close()
ethnoCatAttr = "property"
ethnoTagAttr = "resource"
ethnoCleanCat = lambda x: x.replace('eoe:', '')
ethnoCleanTag = lambda x: x.replace('[eoe:', '').replace(']', '')
def chunkPost(post):
if not post:
return []
tagList = []
soup = BeautifulSoup(post)
spans = soup.find_all('span')
for s in spans:
#print s.attrs
category = ""
tag = ""
if ethnoCatAttr in s.attrs:
category = ethnoCleanCat(s[ethnoCatAttr])
if ethnoTagAttr in s.attrs:
tag = ethnoCleanTag(s[ethnoTagAttr])
if category != "" and tag != "":
fillMapArray(category, tag, categoryToTag)
if tag != "":
tagList.append(tag)
return set(tagList)
def loadEthnoPosts():
f = open(stfPath+stfEthnoPost)
dpNodes = json.load(f)
for dpN in dpNodes[nodesKey]:
loadOneEthnoPost(dpN)
f.close()
def main(graph):
loadEthnoPosts()
loadEthnoComments()
for t in categoryToTag:
categoryToTag[t] = [i for i in set(categoryToTag[t])]
print categoryToTag
export = json.dumps(categoryToTag)
f = open(stfPath+"categoryToTag.json", "w")
f.write(export)
f.close()