split into 2 diff files and use concurrent thread

.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+*.env
+venv

apigw/qrcode-apigw-api-swagger-apigateway.json

@@ -0,0 +1,610 @@
+{
+  "swagger" : "2.0",
+  "info" : {
+    "version" : "2024-07-13T15:48:52Z",
+    "title" : "qrcode-apigw"
+  },
+  "host" : "bk5wiynzsi.execute-api.ap-southeast-1.amazonaws.com",
+  "basePath" : "/api",
+  "schemes" : [ "https" ],
+  "paths" : {
+    "/v1/gmail/getEmails" : {
+      "get" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "GET",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/gmail/getEmails",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/qrcodetypes/checkRedirects" : {
+      "post" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "POST",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/checkRedirects",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/qrcodetypes/detect" : {
+      "post" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "POST",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/detect",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/qrcodetypes/getQRDetails" : {
+      "get" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "GET",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/getQRDetails",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/qrcodetypes/scan" : {
+      "post" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "POST",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/scan",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/qrcodetypes/verifyURL" : {
+      "post" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "POST",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/verifyURL",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/qrcodetypes/virusTotalCheck" : {
+      "post" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "POST",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/virusTotalCheck",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/deleteAllBookmark" : {
+      "put" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "PUT",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteAllBookmark",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/deleteAllScannedHistories" : {
+      "put" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "PUT",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteAllScannedHistories",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/deleteBookmark" : {
+      "put" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "PUT",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteBookmark",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+            
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/deleteScannedHistories" : {
+      "put" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "PUT",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteScannedHistories",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/getBookmarks" : {
+      "get" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "GET",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/getBookmarks",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/getScannedHistories" : {
+      "get" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "GET",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/getScannedHistories",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/getUser" : {
+      "get" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "GET",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/getUser",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/user/setBookmark" : {
+      "post" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "security" : [ {
+          "Cognito" : [ ]
+        } ],
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "POST",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/setBookmark",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "requestParameters" : {
+            "integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    },
+    "/v1/version" : {
+      "get" : {
+        "produces" : [ "application/json" ],
+        "responses" : {
+          "200" : {
+            "description" : "200 response",
+            "schema" : {
+              "$ref" : "#/definitions/Empty"
+            }
+          }
+        },
+        "x-amazon-apigateway-integration" : {
+          "connectionId" : "h1icfc",
+          "httpMethod" : "GET",
+          "uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/version",
+          "responses" : {
+            "default" : {
+              "statusCode" : "200"
+            }
+          },
+          "passthroughBehavior" : "when_no_match",
+          "connectionType" : "VPC_LINK",
+          "tlsConfig" : {
+            "insecureSkipVerification" : true
+          },
+          "type" : "http_proxy"
+        }
+      }
+    }
+  },
+  "securityDefinitions" : {
+    "Cognito" : {
+      "type" : "apiKey",
+      "name" : "Authorization",
+      "in" : "header",
+      "x-amazon-apigateway-authtype" : "cognito_user_pools",
+      "x-amazon-apigateway-authorizer" : {
+        "providerARNs" : [ "arn:aws:cognito-idp:ap-southeast-1:058264269465:userpool/ap-southeast-1_zkxA0NZYN" ],
+        "type" : "cognito_user_pools"
+      }
+    }
+  },
+  "definitions" : {
+    "Empty" : {
+      "type" : "object",
+      "title" : "Empty Schema"
+    }
+  },
+  "x-amazon-apigateway-gateway-responses" : {
+    "DEFAULT_4XX" : {
+      "responseParameters" : {
+        "gatewayresponse.header.Access-Control-Allow-Methods" : "'OPTIONS'",
+        "gatewayresponse.header.Access-Control-Allow-Origin" : "'*'",
+        "gatewayresponse.header.Access-Control-Allow-Headers" : "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'"
+      }
+    },
+    "DEFAULT_5XX" : {
+      "responseParameters" : {
+        "gatewayresponse.header.Access-Control-Allow-Methods" : "'OPTIONS'",
+        "gatewayresponse.header.Access-Control-Allow-Origin" : "'*'",
+        "gatewayresponse.header.Access-Control-Allow-Headers" : "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'"
+      }
+    }
+  },
+  "x-amazon-apigateway-policy" : {
+    "Version" : "2012-10-17",
+    "Statement" : [ {
+      "Effect" : "Allow",
+      "Principal" : "*",
+      "Action" : "execute-api:Invoke",
+      "Resource" : "arn:aws:execute-api:ap-southeast-1:058264269465:bk5wiynzsi/*/*/*"
+    } ]
+  }
+}

dataset/load_data.py

@@ -0,0 +1,63 @@
+import csv
+import os
+import requests
+import concurrent.futures
+
+# Define the endpoint URL
+endpoint_url = "http://localhost:8080/v1/qrcodetypes/scan" 
+
+# Path to the CSV file
+csv_file_path = "malicious_phish.csv"
+
+# Directory to store the split CSV files
+split_files_dir = "split_csv_files"
+os.makedirs(split_files_dir, exist_ok=True)
+
+# Function to ensure URL starts with http:// or https://
+def ensure_url_prefix(url):
+    if not (url.startswith("http://") or url.startswith("https://")):
+        return "https://" + url
+    return url
+
+# Read the CSV file and split into 20 files
+def split_csv_file(csv_file_path, split_files_dir, num_splits=20):
+    with open(csv_file_path, newline='') as csvfile:
+        reader = list(csv.DictReader(csvfile))
+        total_rows = len(reader)
+        rows_per_file = total_rows // num_splits
+        
+        for i in range(num_splits):
+            split_file_path = os.path.join(split_files_dir, f"split_file_{i+1}.csv")
+            with open(split_file_path, 'w', newline='') as split_file:
+                writer = csv.DictWriter(split_file, fieldnames=['url', 'type'])
+                writer.writeheader()
+                start_index = i * rows_per_file
+                end_index = (i + 1) * rows_per_file if i != num_splits - 1 else total_rows
+                for row in reader[start_index:end_index]:
+                    row['url'] = ensure_url_prefix(row['url'])
+                    writer.writerow(row)
+
+# Function to process a CSV file and send POST requests
+def process_csv_file(csv_file_path):
+    with open(csv_file_path, newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            url = row['url']  # Column header for URL is 'url'
+            response = requests.post(endpoint_url, json={"data": url})
+            if response.status_code == 200:
+                print(f"Successfully sent data: {url}")
+            else:
+                print(f"Failed to send data: {url}, Status code: {response.status_code}")
+
+# Split the original CSV file into 20 parts
+split_csv_file(csv_file_path, split_files_dir)
+
+# Get the list of split CSV files
+split_files = [os.path.join(split_files_dir, file) for file in os.listdir(split_files_dir) if file.endswith('.csv')]
+
+# Execute the requests concurrently
+with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
+    futures = [executor.submit(process_csv_file, split_file) for split_file in split_files]
+    concurrent.futures.wait(futures)
+
+print("Processing completed.")

gmail-json/test.json

@@ -0,0 +1,134 @@
+{
+    "emails": [
+        {
+            "snippet": "",
+            "threadId": "190e95e29f2334d8",
+            "labelIds": [
+                "IMPORTANT",
+                "SENT",
+                "INBOX"
+            ],
+            "payload": {
+                "parts": [
+                    {
+                        "parts": [
+                            {
+                                "headers": [
+                                    {
+                                        "name": "Content-Type",
+                                        "value": "text/plain; charset=\"UTF-8\""
+                                    }
+                                ],
+                                "filename": "",
+                                "mimeType": "text/plain",
+                                "body": {
+                                    "data": "DQo=",
+                                    "size": 2
+                                },
+                                "partId": "0.0"
+                            },
+                            {
+                                "headers": [
+                                    {
+                                        "name": "Content-Type",
+                                        "value": "text/html; charset=\"UTF-8\""
+                                    }
+                                ],
+                                "filename": "",
+                                "mimeType": "text/html",
+                                "body": {
+                                    "data": "PGRpdj48aW1nIHNyYz0iY2lkOjE5MGU5NWYzZDAwOTZmZjQ5YjQxIiBzdHlsZT0ibWF4LXdpZHRoOiAxMDAlOyI-PC9kaXY-DQo=",
+                                    "size": 74
+                                },
+                                "partId": "0.1"
+                            }
+                        ],
+                        "headers": [
+                            {
+                                "name": "Content-Type",
+                                "value": "multipart/alternative; boundary=\"0000000000006b7c83061e0f9c47\""
+                            }
+                        ],
+                        "filename": "",
+                        "mimeType": "multipart/alternative",
+                        "body": {
+                            "size": 0
+                        },
+                        "partId": "0"
+                    },
+                    {
+                        "headers": [
+                            {
+                                "name": "Content-Type",
+                                "value": "image/png; name=\"6E0889F8-6DAD-417C-A1AF-756DBC1138AC.png\""
+                            },
+                            {
+                                "name": "Content-Disposition",
+                                "value": "attachment; filename=\"6E0889F8-6DAD-417C-A1AF-756DBC1138AC.png\""
+                            },
+                            {
+                                "name": "Content-Transfer-Encoding",
+                                "value": "base64"
+                            },
+                            {
+                                "name": "X-Attachment-Id",
+                                "value": "190e95f3d0096ff49b41"
+                            },
+                            {
+                                "name": "Content-ID",
+                                "value": "<190e95f3d0096ff49b41>"
+                            }
+                        ],
+                        "filename": "6E0889F8-6DAD-417C-A1AF-756DBC1138AC.png",
+                        "mimeType": "image/png",
+                        "body": {
+                            "size": 579220,
+                            "attachmentId": "ANGjdJ-HCTgEgqdJXIljB63MrOfI7vChyoOoesaB3u3MGzB038Gv47OSpXQ8DB8l4IpQ8uowl7BUys7Q2dQDv46AylseBwXr4tKmqCJ9DSOFcQnYqOUqN-4t_tlQf7jpI5IoYgpRkqCk6Hr_lihjzfyxSqRGVIgVHJgUvCEJT6jgBEBDDEHqFI8y2JcXyYEuzE51_BCUWTQOUcu6PvVzJSCSvKoLc3zzwYfsMHXr4UX1lSIHAKxDTvfYj-m0KzfPucrcl0ywLD5CvgIJUGsE1X6tbgRRfoHfwL9wdSQHsedre0t_sXNlY592jcy9pUs"
+                        },
+                        "partId": "1"
+                    }
+                ],
+                "headers": [
+                    {
+                        "name": "MIME-Version",
+                        "value": "1.0"
+                    },
+                    {
+                        "name": "Date",
+                        "value": "Thu, 25 Jul 2024 18:10:57 +0800"
+                    },
+                    {
+                        "name": "Message-ID",
+                        "value": "<CAOhDyE09kVfVpYTOwm6cwir9tLaYvs9JM-4ZJsNQpk57uOOd_w@mail.gmail.com>"
+                    },
+                    {
+                        "name": "Subject",
+                        "value": "Test Test"
+                    },
+                    {
+                        "name": "From",
+                        "value": "SafeQR <safeqr.fyp.24.s2.43p@gmail.com>"
+                    },
+                    {
+                        "name": "To",
+                        "value": "SafeQR <safeqr.fyp.24.s2.43p@gmail.com>"
+                    },
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/related; boundary=\"0000000000006b7c84061e0f9c48\""
+                    }
+                ],
+                "filename": "",
+                "mimeType": "multipart/related",
+                "body": {
+                    "size": 0
+                },
+                "partId": ""
+            },
+            "historyId": 27133,
+            "id": "190e95f5626b2a39",
+            "sizeEstimate": 793666,
+            "internalDate": 1721902257000
+        }
+    ]
+}

gmail-json/test_tree.json

@@ -0,0 +1,11 @@
+{
+    "cartoon": {
+        "0": {
+            "more than 1 person": {
+                "0": "no",
+                "1": { "winter": { "0": "no", "1": "yes" } }
+            }
+        },
+        "1": "no"
+    }
+}

serverless/safeqr-signup-post-confirmation-lambda.py

@@ -0,0 +1,69 @@
+import os
+import pg8000
+
+def lambda_handler(event, context):
+    print(f"Event: {event}")
+    
+    # Extract user attributes from the Cognito event
+    user_attributes = event['request']['userAttributes']
+    
+    # Extract specific attributes (modify as needed)
+    user_id = user_attributes['sub']
+    email = user_attributes['email']
+    name = user_attributes.get('name', 'Default User')
+    
+    # Extract providerName from identities
+    identities = user_attributes.get('identities', [])
+    provider_name = identities[0]['providerName'] if identities else 'DefaultApp'
+    print(f"providerName: {provider_name}")
+    
+    # Database connection parameters
+    db_host = os.environ['RDS_HOST']
+    db_name = os.environ['RDS_DB_NAME']
+    db_user = os.environ['RDS_USER']
+    db_password = os.environ['RDS_PASSWORD']
+    db_port = 5432
+    
+    # Connect to the database
+    try:
+        conn = pg8000.connect(
+            host=db_host,
+            port=db_port,
+            database=db_name,
+            user=db_user,
+            password=db_password
+        )
+        
+        # Create a cursor
+        cur = conn.cursor()
+        print("Connection to db success!")
+        
+        # SQL query to insert user details
+        insert_query = """
+        INSERT INTO safeqr."user" (id, email, name, roles, source)
+        VALUES (%s, %s, %s, %s, %s)
+        ON CONFLICT (id) DO UPDATE
+        SET email = EXCLUDED.email, name = EXCLUDED.name, source = EXCLUDED.source;
+        """
+        
+        # Execute the query
+        cur.execute(insert_query, (user_id, email, name, ['appuser'], provider_name))
+        
+        # Commit the transaction
+        conn.commit()
+        
+        print(f"User {user_id} inserted/updated successfully")
+        
+    except Exception as e:
+        print(f"Database error: {str(e)}")
+        raise e
+    
+    finally:
+        # Close the cursor and connection
+        if 'cur' in locals():
+            cur.close()
+        if 'conn' in locals():
+            conn.close()
+    
+    # Return the event object back to Cognito
+    return event

sql/Create_all_tables.sql

@@ -0,0 +1,265 @@
+-- Create schema if it doesn't exist
+CREATE SCHEMA IF NOT EXISTS safeqr;
+
+
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Create Enum for scan_history status
+CREATE TYPE scan_status AS ENUM ('ACTIVE', 'INACTIVE');
+
+-- safeqr."user" definition
+
+-- Drop table
+
+-- DROP TABLE safeqr."user";
+
+CREATE TABLE safeqr."user" (
+	id varchar(255) NOT NULL,
+	"name" varchar(255) NULL,
+	email varchar(255) NULL,
+	"source" varchar(255) NULL,
+	date_created timestamptz DEFAULT now() NULL,
+	date_updated timestamptz DEFAULT now() NULL,
+	roles _text NULL,
+	status varchar(255) DEFAULT 'ACTIVE'::character varying NULL,
+	gm
+
+-- safeqr.qr_code_types definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.qr_code_types;
+
+CREATE TABLE safeqr.qr_code_types (
+	id bigserial NOT NULL,
+	"type" varchar(255) NOT NULL,
+	description varchar(255) NULL,
+	prefix varchar(255) NULL,
+	table_name varchar(255) NULL,
+	CONSTRAINT qr_code_types_pkey PRIMARY KEY (id)
+);
+
+-- safeqr.qr_code definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.qr_code;
+
+CREATE TABLE safeqr.qr_code (
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	qr_code_type_id int8 NULL,
+	user_id varchar(255) NULL,
+	contents text NULL,
+	created_at timestamptz DEFAULT CURRENT_TIMESTAMP NULL,
+	CONSTRAINT qr_code_pkey PRIMARY KEY (id),
+	CONSTRAINT qr_code_qr_code_type_id_fkey FOREIGN KEY (qr_code_type_id) REFERENCES safeqr.qr_code_types(id),
+	CONSTRAINT qr_code_user_id_fkey FOREIGN KEY (user_id) REFERENCES safeqr."user"(id)
+);
+
+-- safeqr.scan_history definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.scan_history;
+
+CREATE TABLE safeqr.scan_history (
+	id bigserial NOT NULL,
+	qr_code_id uuid NULL,
+	user_id varchar(255) NULL,
+	status varchar(255) DEFAULT 'ACTIVE'::safeqr.scan_status NULL,
+	date_created timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	date_updated timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	bookmarked bool DEFAULT false NULL,
+	CONSTRAINT scan_history_pkey PRIMARY KEY (id),
+	CONSTRAINT scan_history_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- safeqr.scan_bookmark definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.scan_bookmark;
+
+CREATE TABLE safeqr.scan_bookmark (
+	id bigserial NOT NULL,
+	qr_code_id uuid NULL,
+	user_id varchar(255) NULL,
+	status varchar(255) DEFAULT 'ACTIVE'::safeqr.bookmark_status NULL,
+	date_created timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	date_updated timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	CONSTRAINT scan_bookmark_pkey PRIMARY KEY (id),
+	CONSTRAINT scan_bookmark_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- safeqr.url definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.url;
+
+CREATE TABLE safeqr.url (
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	qr_code_id uuid NULL,
+	"domain" text NULL,
+	subdomain text NULL,
+	top_level_domain text NULL,
+	query text NULL,
+	fragment text NULL,
+	redirect int4 DEFAULT 0 NULL,
+	"path" text NULL,
+	redirect_chain _text NULL,
+	hsts_header _text NULL,
+	ssl_stripping _bool NULL,
+	CONSTRAINT url_pkey PRIMARY KEY (id),
+	CONSTRAINT url_qr_code_id_fkey FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id)
+);
+
+-- safeqr."text" definition
+
+-- Drop table
+
+-- DROP TABLE safeqr."text";
+
+CREATE TABLE safeqr."text" (
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	qr_code_id uuid NULL,
+	"text" varchar(2048) NULL,
+	CONSTRAINT text_pkey PRIMARY KEY (id),
+	CONSTRAINT text_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- safeqr.phone definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.phone;
+
+CREATE TABLE safeqr.phone (
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	qr_code_id uuid NULL,
+	phone text NULL,
+	CONSTRAINT phone_pkey PRIMARY KEY (id),
+	CONSTRAINT phone_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- safeqr.sms definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.sms;
+
+CREATE TABLE safeqr.sms (
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	qr_code_id uuid NULL,
+	phone text NULL,
+	message text NULL,
+	CONSTRAINT sms_pkey PRIMARY KEY (id),
+	CONSTRAINT sms_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id)
+);
+
+-- Create Email table
+CREATE TABLE safeqr.email (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    qr_code_id UUID REFERENCES safeqr.qr_code(id),
+    email VARCHAR(200),
+    title VARCHAR(200),
+    message VARCHAR(7089)
+);
+
+-- Create GeoLocation table
+CREATE TABLE safeqr.geolocation (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    qr_code_id UUID REFERENCES safeqr.qr_code(id),
+    latitude DOUBLE PRECISION,
+    longitude DOUBLE PRECISION
+);
+
+-- Create Wifi table
+CREATE TABLE safeqr.wifi (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    qr_code_id UUID REFERENCES safeqr.qr_code(id),
+    ssid VARCHAR(200),
+    password VARCHAR(200),
+    encryption VARCHAR(200),
+    hidden BOOLEAN
+);
+
+-- Create Bitcoin table
+CREATE TABLE safeqr.bitcoin (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    qr_code_id UUID REFERENCES safeqr.qr_code(id),
+    address VARCHAR(200),
+    amount BIGINT,
+    message VARCHAR(250)
+);
+
+-- Create VCard table
+CREATE TABLE safeqr.vcard (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    qr_code_id UUID REFERENCES safeqr.qr_code(id),
+    version VARCHAR(50),
+    name VARCHAR(250),
+    organisation VARCHAR(250),
+    title VARCHAR(50),
+    address VARCHAR(250),
+    tel_work VARCHAR(250),
+    tel_cell VARCHAR(250),
+    email VARCHAR(250),
+    url VARCHAR(2048)
+);
+
+-- safeqr.gmail_emails definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.gmail_emails;
+
+CREATE TABLE safeqr.gmail_emails (
+	user_id varchar NOT NULL,
+	subject text NULL,
+	date_received timestamptz NULL,
+	date_created timestamptz DEFAULT CURRENT_TIMESTAMP NULL,
+	message_id varchar NOT NULL,
+	history_id int8 NULL,
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	CONSTRAINT gmail_pk PRIMARY KEY (id),
+	CONSTRAINT gmail_unique UNIQUE (user_id, message_id),
+	CONSTRAINT gmail_user_fk FOREIGN KEY (user_id) REFERENCES safeqr."user"(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- safeqr.gmail_urls definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.gmail_urls;
+
+CREATE TABLE safeqr.gmail_urls (
+	gmail_id uuid NOT NULL,
+	image_url text NOT NULL,
+	decoded_content varchar NULL,
+	qr_code_id uuid NOT NULL,
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	CONSTRAINT gmail_urls_pk PRIMARY KEY (id),
+	CONSTRAINT gmail_urls_unique UNIQUE (gmail_id, image_url, decoded_content),
+	CONSTRAINT gmail_urls_gmail_emails_fk FOREIGN KEY (gmail_id) REFERENCES safeqr.gmail_emails(id) ON DELETE CASCADE ON UPDATE CASCADE,
+	CONSTRAINT gmail_urls_qr_code_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- safeqr.gmail_cid definition
+
+-- Drop table
+
+-- DROP TABLE safeqr.gmail_cid;
+
+CREATE TABLE safeqr.gmail_cid (
+	gmail_id uuid NOT NULL,
+	cid varchar NOT NULL,
+	attachment_id text NOT NULL,
+	decoded_content text NOT NULL,
+	qr_code_id uuid NOT NULL,
+	id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
+	CONSTRAINT gmail_cid_pk PRIMARY KEY (id),
+	CONSTRAINT gmail_cid_unique UNIQUE (gmail_id, cid, decoded_content),
+	CONSTRAINT gmail_cid_gmail_emails_fk FOREIGN KEY (gmail_id) REFERENCES safeqr.gmail_emails(id) ON DELETE CASCADE ON UPDATE CASCADE,
+	CONSTRAINT gmail_cid_qr_code_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
+);

sql/Drop_all_tables.sql

@@ -0,0 +1,16 @@
+-- Drop the tables with CASCADE to handle dependencies
+
+DROP TABLE IF EXISTS safeqr.vcard CASCADE;
+DROP TABLE IF EXISTS safeqr.bitcoin CASCADE;
+DROP TABLE IF EXISTS safeqr.wifi CASCADE;
+DROP TABLE IF EXISTS safeqr.geolocation CASCADE;
+DROP TABLE IF EXISTS safeqr.email CASCADE;
+DROP TABLE IF EXISTS safeqr.sms CASCADE;
+DROP TABLE IF EXISTS safeqr.phone CASCADE;
+DROP TABLE IF EXISTS safeqr.text CASCADE;
+DROP TABLE IF EXISTS safeqr.url CASCADE;
+DROP TABLE IF EXISTS safeqr.scan_bookmark CASCADE;
+DROP TABLE IF EXISTS safeqr.scan_history CASCADE;
+DROP TABLE IF EXISTS safeqr.qr_code CASCADE;
+DROP TABLE IF EXISTS safeqr.qr_code_types CASCADE;
+DROP TABLE IF EXISTS safeqr."user" CASCADE;

sql/Dummy_data.sql

@@ -0,0 +1,97 @@
+
+INSERT INTO safeqr."user" (
+    id, 
+    cognitoid, 
+    firstname, 
+    lastname, 
+    email, 
+    source, 
+    password, 
+    salt, 
+    cognito_id, 
+    first_name, 
+    last_name
+) VALUES (
+    'test-unique-id', 
+    'test-cognito-id', 
+    'Test', 
+    'User', 
+    'test.user@example.com', 
+    'test-source', 
+    'test-password-hash', 
+    'test-salt', 
+    NULL, 
+    NULL, 
+    NULL
+);
+
+
+-- Insert into QR_Code_Types table
+-- Insert additional QR code types into the qr_code_types table
+INSERT INTO safeqr.qr_code_types (type, description, prefix, table_name) VALUES 
+('URL', 'Uniform Resource Locator', 'http://', 'url'),
+('URL', 'Uniform Resource Locator', 'https://', 'url'),
+('EMAIL', 'Email Address', 'mailto:', 'email'),
+('PHONE', 'Phone Number', 'tel:', 'phone'),
+('SMS', 'Short Message Service', 'smsto:', 'sms'),
+('GEOLOCATION', 'Geographic Location', 'geo:', 'geolocation'),
+('WIFI', 'Wireless Network Configuration', 'WIFI:', 'wifi'),
+('BITCOIN', 'Cryptocurrency Address', 'bitcoin:', 'bitcoin'),
+('VCARD', 'Virtual Contact File', 'BEGIN:VCARD', 'vcard'),
+('TEXT', 'Text', '', 'text');
+
+
+-- Insert into QR_Code table
+INSERT INTO safeqr.qr_code (id, qr_code_type_id, user_id, contents, created_at) VALUES
+(gen_random_uuid(), 1, 'test-unique-id', 'http://example.com', CURRENT_TIMESTAMP),
+(gen_random_uuid(), 2, NULL, 'https://example.com', CURRENT_TIMESTAMP),
+(gen_random_uuid(), 3, 'test-unique-id', 'mailto:test@example.com', CURRENT_TIMESTAMP);
+
+-- Insert into Scan_History table
+INSERT INTO safeqr.scan_history (qr_code_id, user_id) VALUES
+((SELECT id FROM safeqr.qr_code LIMIT 1), 'test-unique-id'),
+((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 1), 'test-unique-id'),
+((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 2), 'test-unique-id');
+
+-- Insert into Scan_Bookmark table
+INSERT INTO safeqr.scan_bookmark (qr_code_id, user_id, active) VALUES
+((SELECT id FROM safeqr.qr_code LIMIT 1), 'test-unique-id', 'ACTIVE'),
+((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 1), 'test-unique-id', 'ACTIVE'),
+((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 2), 'test-unique-id', 'DELETED');
+
+-- Insert into URL table
+INSERT INTO safeqr.url (id, qr_code_id, url, query, fragment) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents LIKE 'http://%' LIMIT 1), 'http://example.com', 'key=value', 'fragment'),
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents LIKE 'https://%' LIMIT 1), 'https://example.com', 'key=value', 'fragment');
+
+-- Insert into Text table
+INSERT INTO safeqr.text (id, qr_code_id, text) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents = 'Some text' LIMIT 1), 'Some example text');
+
+-- Insert into Phone table
+INSERT INTO safeqr.phone (id, qr_code_id, phone) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '+1234567890');
+
+-- Insert into SMS table
+INSERT INTO safeqr.sms (id, qr_code_id, phone, message) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '+1234567890', 'This is a test SMS message.');
+
+-- Insert into Email table
+INSERT INTO safeqr.email (id, qr_code_id, email, title, message) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents LIKE 'mailto:%' LIMIT 1), 'test@example.com', 'Test Email', 'This is a test email body.');
+
+-- Insert into GeoLocation table
+INSERT INTO safeqr.geolocation (id, qr_code_id, latitude, longitude) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), 37.7749, -122.4194);
+
+-- Insert into Wifi table
+INSERT INTO safeqr.wifi (id, qr_code_id, ssid, password, encryption, hidden) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), 'TestSSID', 'password123', 'WPA2', FALSE);
+
+-- Insert into Bitcoin table
+INSERT INTO safeqr.bitcoin (id, qr_code_id, address, amount, message) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', 5000000000, 'Test Bitcoin transaction');
+
+-- Insert into VCard table
+INSERT INTO safeqr.vcard (id, qr_code_id, version, name, organisation, title, address, tel_work, tel_cell, email, url) VALUES
+(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '4.0', 'John Doe', 'Example Corp', 'Developer', '1234 Elm St, Springfield, IL', '+1234567890', '+0987654321', 'johndoe@example.com', 'http://example.com');

sql/qr_code_types_202408032138.sql

@@ -0,0 +1,11 @@
+INSERT INTO safeqr.qr_code_types ("type",description,prefix,table_name) VALUES
+	 ('URL','Uniform Resource Locator','http://','url'),
+	 ('EMAIL','Email Address','mailto:','email'),
+	 ('PHONE','Phone Number','tel:','phone'),
+	 ('SMS','Short Message Service','smsto:','sms'),
+	 ('GEOLOCATION','Geographic Location','geo:','geolocation'),
+	 ('WIFI','Wireless Network Configuration','WIFI:','wifi'),
+	 ('BITCOIN','Cryptocurrency Address','bitcoin:','bitcoin'),
+	 ('VCARD','Virtual Contact File','BEGIN:VCARD','vcard'),
+	 ('URL','Secure Uniform Resource Locator','https://','url'),
+	 ('TEXT','Text','','text');