Upload New File

Template.py

+import json
+import sys
+import boto3
+from boto3.s3.transfer import S3Transfer
+from abc import ABCMeta
+
+
+class Baseclass(metaclass=ABCMeta):
+
+    def metadata_setup(self, cong_file):
+        """
+        Setup metadata for the application
+        :return: set of global variables
+        """
+        try:
+            f = open(f"metadata/{cong_file}.json", "r")
+            data = json.loads(f.read())
+            self.output_file = data["output_file"]
+
+            # twitter configuration
+            if 'twitter' in cong_file:
+                self.key_word = data["key_word"]
+                self.access_token = data["access_token"]
+                self.data_size = data["data_size"]
+            elif 'quotes' in cong_file:
+                self.base_url = data["base_url"]
+
+            # snowflake configuration
+            self.Account = data["snowflake"]["Account"]
+            self.data_warehouse = data["snowflake"]["data_warehouse"]
+            self.username = data["snowflake"]["username"]
+            self.password = data["snowflake"]["password"]
+            self.role = data["snowflake"]["role"]
+            self.database = data["snowflake"]["database"]
+            self.schema = data["snowflake"]["schema"]
+            self.table_name = data["snowflake"]["table_name"]
+
+            # aws configuration
+            self.access_key = data["aws"]["aws_access_key_id"]
+            self.secret_key = data["aws"]["aws_secret_access_key"]
+            self.bucket = data["aws"]["bucket"]
+
+        except Exception as e:
+            print(e)
+            sys.exit(-1)
+        else:
+            print("Metadata configuration success")
+
+    def s3_data_transfer(self, file):
+        """
+        Extract files will be uploaded to aws bucket
+        :param file: file which needs to be uploaded
+        :return: No return values
+        """
+        try:
+            client = boto3.client('s3',
+                                  aws_access_key_id=self.access_key,
+                                  aws_secret_access_key=self.secret_key,
+                                  )
+            s3 = boto3.resource('s3')
+            transfer = S3Transfer(client)
+            transfer.upload_file(file, self.bucket, file)
+        except Exception as e:
+            print(e)
+            sys.exit(-1)
+        else:
+            print("Data successfully load to s3 bucket")
\ No newline at end of file