Commit eea740ad authored by M.S.M.Perera's avatar M.S.M.Perera

Implemented hadoop upload function

parent 6d0e9797
...@@ -384,6 +384,23 @@ def PdfBreaker(gz_url): ...@@ -384,6 +384,23 @@ def PdfBreaker(gz_url):
return js return js
def hadoopUpload(jsObj, gazetteno, exType, pubDate):
client = InsecureClient('http://localhost:9870',
user='Shalini')
data__ = []
content = None
if exType == 0:
outputFileName = '/home/user_e/egaz/gz-p-' + \
str(gazetteno) + '.json'
elif exType == 1:
outputFileName = '/home/user_e/egaz/gz-e-' + \
str(gazetteno) + '.json'
client.write(outputFileName,
data=dumps(jsObj), overwrite=True)
def readHadoopDataAll(): def readHadoopDataAll():
data_ = [] data_ = []
...@@ -412,21 +429,7 @@ def readHadoopDataAll(): ...@@ -412,21 +429,7 @@ def readHadoopDataAll():
return data_ return data_
def hadoopUpload(jsObj, gazetteno, exType, pubDate):
client = InsecureClient('http://localhost:9870',
user='Shalini')
data__ = []
content = None
if exType == 0:
outputFileName = '/home/user_e/egaz/gz-p-' + \
str(gazetteno) + '.json'
elif exType == 1:
outputFileName = '/home/user_e/egaz/gz-e-' + \
str(gazetteno) + '.json'
client.write(outputFileName,
data=dumps(jsObj), overwrite=True)
# data_ = readHadoopDataVacancy() # data_ = readHadoopDataVacancy()
# data_ = TopicCluster.clustervac(data_) # data_ = TopicCluster.clustervac(data_)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment