Commit 6d0e9797 authored by M.S.M.Perera's avatar M.S.M.Perera

Implemented read hadoop data function

parent befff334
...@@ -384,32 +384,6 @@ def PdfBreaker(gz_url): ...@@ -384,32 +384,6 @@ def PdfBreaker(gz_url):
return js return js
def hadoopUpload(jsObj, gazetteno, exType, pubDate):
client = InsecureClient('http://localhost:9870',
user='Shalini')
data__ = []
content = None
if exType == 0:
outputFileName = '/home/user_e/egaz/gz-p-' + \
str(gazetteno) + '.json'
elif exType == 1:
outputFileName = '/home/user_e/egaz/gz-e-' + \
str(gazetteno) + '.json'
client.write(outputFileName,
data=dumps(jsObj), overwrite=True)
# data_ = readHadoopDataVacancy()
# data_ = TopicCluster.clustervac(data_)
# for item in data_:
# if item["category"] == id:
# data__.append(item)
# new = Vac(department=dep, title=ct, description=completeDescription, summary=summarizedDescription, date=pubDate, url=gz_url, cat=)
# new.save()
def readHadoopDataAll(): def readHadoopDataAll():
data_ = [] data_ = []
...@@ -438,6 +412,33 @@ def readHadoopDataAll(): ...@@ -438,6 +412,33 @@ def readHadoopDataAll():
return data_ return data_
def hadoopUpload(jsObj, gazetteno, exType, pubDate):
client = InsecureClient('http://localhost:9870',
user='Shalini')
data__ = []
content = None
if exType == 0:
outputFileName = '/home/user_e/egaz/gz-p-' + \
str(gazetteno) + '.json'
elif exType == 1:
outputFileName = '/home/user_e/egaz/gz-e-' + \
str(gazetteno) + '.json'
client.write(outputFileName,
data=dumps(jsObj), overwrite=True)
# data_ = readHadoopDataVacancy()
# data_ = TopicCluster.clustervac(data_)
# for item in data_:
# if item["category"] == id:
# data__.append(item)
# new = Vac(department=dep, title=ct, description=completeDescription, summary=summarizedDescription, date=pubDate, url=gz_url, cat=)
# new.save()
def readHadoopDataExam(): def readHadoopDataExam():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment