Commit e77a4cc9 authored by M.S.M.Perera's avatar M.S.M.Perera

Upload read secondary storage function

parent 5ad2e8a8
...@@ -442,6 +442,19 @@ def SingleUpload(jsObj, name): ...@@ -442,6 +442,19 @@ def SingleUpload(jsObj, name):
client.write(outputFileName, data=dumps(jsObj), overwrite=True) client.write(outputFileName, data=dumps(jsObj), overwrite=True)
def readSecondaryStorage():
data_ = []
path_to_json = 'dumps/well-structured-final/'
for file_name in [file for file in os.listdir(path_to_json) if file.endswith('.json')]:
with open(path_to_json + file_name, encoding='utf-8') as json_file:
data = json.load(json_file)
data_.extend(data)
print(data)
return data_
# data_ = readHadoopDataVacancy() # data_ = readHadoopDataVacancy()
# data_ = TopicCluster.clustervac(data_) # data_ = TopicCluster.clustervac(data_)
# for item in data_: # for item in data_:
...@@ -487,18 +500,6 @@ def readHadoopDataVacancy(): ...@@ -487,18 +500,6 @@ def readHadoopDataVacancy():
return data_ return data_
def readSecondaryStorage():
data_ = []
path_to_json = 'dumps/well-structured-final/'
for file_name in [file for file in os.listdir(path_to_json) if file.endswith('.json')]:
with open(path_to_json + file_name, encoding='utf-8') as json_file:
data = json.load(json_file)
data_.extend(data)
print(data)
return data_
# API configuarations # API configuarations
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment