isky/Database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rebranch

Browse Source
This commit is contained in:
heyethereum
2024-08-11 07:58:19 +08:00
commit 1625de948f
21 changed files with 1374546 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
.DS_Store
*.env
venv

1
README.md Normal file
View File

@@ -0,0 +1 @@
# Database

610
apigw/qrcode-apigw-api-swagger-apigateway.json Normal file
View File

@@ -0,0 +1,610 @@
{
"swagger" : "2.0",
"info" : {
"version" : "2024-07-13T15:48:52Z",
"title" : "qrcode-apigw"
},
"host" : "bk5wiynzsi.execute-api.ap-southeast-1.amazonaws.com",
"basePath" : "/api",
"schemes" : [ "https" ],
"paths" : {
"/v1/gmail/getEmails" : {
"get" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "GET",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/gmail/getEmails",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/qrcodetypes/checkRedirects" : {
"post" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "POST",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/checkRedirects",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/qrcodetypes/detect" : {
"post" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "POST",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/detect",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/qrcodetypes/getQRDetails" : {
"get" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "GET",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/getQRDetails",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/qrcodetypes/scan" : {
"post" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "POST",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/scan",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/qrcodetypes/verifyURL" : {
"post" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "POST",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/verifyURL",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/qrcodetypes/virusTotalCheck" : {
"post" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "POST",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/qrcodetypes/virusTotalCheck",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/deleteAllBookmark" : {
"put" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "PUT",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteAllBookmark",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/deleteAllScannedHistories" : {
"put" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "PUT",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteAllScannedHistories",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/deleteBookmark" : {
"put" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "PUT",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteBookmark",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/deleteScannedHistories" : {
"put" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "PUT",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/deleteScannedHistories",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/getBookmarks" : {
"get" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "GET",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/getBookmarks",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/getScannedHistories" : {
"get" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "GET",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/getScannedHistories",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/getUser" : {
"get" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "GET",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/getUser",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/user/setBookmark" : {
"post" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"security" : [ {
"Cognito" : [ ]
} ],
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "POST",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/user/setBookmark",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"requestParameters" : {
"integration.request.header.X-USER-ID" : "context.authorizer.claims.sub"
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
},
"/v1/version" : {
"get" : {
"produces" : [ "application/json" ],
"responses" : {
"200" : {
"description" : "200 response",
"schema" : {
"$ref" : "#/definitions/Empty"
}
}
},
"x-amazon-apigateway-integration" : {
"connectionId" : "h1icfc",
"httpMethod" : "GET",
"uri" : "https://safeqr-nlb-6bd79c7ba50f3cb5.elb.ap-southeast-1.amazonaws.com:8443/v1/version",
"responses" : {
"default" : {
"statusCode" : "200"
}
},
"passthroughBehavior" : "when_no_match",
"connectionType" : "VPC_LINK",
"tlsConfig" : {
"insecureSkipVerification" : true
},
"type" : "http_proxy"
}
}
}
},
"securityDefinitions" : {
"Cognito" : {
"type" : "apiKey",
"name" : "Authorization",
"in" : "header",
"x-amazon-apigateway-authtype" : "cognito_user_pools",
"x-amazon-apigateway-authorizer" : {
"providerARNs" : [ "arn:aws:cognito-idp:ap-southeast-1:058264269465:userpool/ap-southeast-1_zkxA0NZYN" ],
"type" : "cognito_user_pools"
}
}
},
"definitions" : {
"Empty" : {
"type" : "object",
"title" : "Empty Schema"
}
},
"x-amazon-apigateway-gateway-responses" : {
"DEFAULT_4XX" : {
"responseParameters" : {
"gatewayresponse.header.Access-Control-Allow-Methods" : "'OPTIONS'",
"gatewayresponse.header.Access-Control-Allow-Origin" : "'*'",
"gatewayresponse.header.Access-Control-Allow-Headers" : "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'"
}
},
"DEFAULT_5XX" : {
"responseParameters" : {
"gatewayresponse.header.Access-Control-Allow-Methods" : "'OPTIONS'",
"gatewayresponse.header.Access-Control-Allow-Origin" : "'*'",
"gatewayresponse.header.Access-Control-Allow-Headers" : "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'"
}
}
},
"x-amazon-apigateway-policy" : {
"Version" : "2012-10-17",
"Statement" : [ {
"Effect" : "Allow",
"Principal" : "*",
"Action" : "execute-api:Invoke",
"Resource" : "arn:aws:execute-api:ap-southeast-1:058264269465:bk5wiynzsi/*/*/*"
} ]
}
}

651207
dataset/concatenated_split_files1.csv Normal file
View File

File diff suppressed because it is too large Load Diff

4775
dataset/failed_requests_2.csv Normal file
View File

File diff suppressed because it is too large Load Diff

6083
dataset/hasExecutable.csv Normal file
View File

File diff suppressed because it is too large Load Diff

12472
dataset/ipadd.csv Normal file
View File

File diff suppressed because it is too large Load Diff

104
dataset/load_data.py Normal file
View File

@@ -0,0 +1,104 @@
import csv
import os
import requests
import concurrent.futures
# Define the endpoint URL
endpoint_url = "http://localhost:8080/v1/qrcodetypes/scan"
# Path to the CSV file
csv_file_path = "hasExecutable.csv"
# Directory to store the split CSV files
split_files_dir = "split_csv_files"
os.makedirs(split_files_dir, exist_ok=True)
# File to store failed requests
failed_requests_file = "failed_requests.csv"
# Final concatenated CSV file
final_concatenated_file = "concatenated_split_files.csv"
# Function to ensure URL starts with http:// or https://
def ensure_url_prefix(url):
if not (url.startswith("http://") or url.startswith("https://")):
return "https://" + url
return url
# Read the CSV file and split into 199 files
def split_csv_file(csv_file_path, split_files_dir, num_splits=199):
with open(csv_file_path, newline='') as csvfile:
reader = list(csv.DictReader(csvfile))
total_rows = len(reader)
rows_per_file = total_rows // num_splits
for i in range(num_splits):
split_file_path = os.path.join(split_files_dir, f"split_file_{i+1}.csv")
with open(split_file_path, 'w', newline='') as split_file:
writer = csv.DictWriter(split_file, fieldnames=['url', 'type'])
writer.writeheader()
start_index = i * rows_per_file
end_index = (i + 1) * rows_per_file if i != num_splits - 1 else total_rows
for row in reader[start_index:end_index]:
row['url'] = ensure_url_prefix(row['url'])
writer.writerow(row)
# Function to process a CSV file and send POST requests
def process_csv_file(csv_file_path):
failed_requests = []
with open(csv_file_path, newline='') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
url = row['url'] # Column header for URL is 'url'
response = requests.post(endpoint_url, json={"data": url})
if response.status_code == 200:
print(f"Successfully sent data: {url}")
else:
print(f"Failed to send data: {url}, Status code: {response.status_code}")
failed_requests.append({"url": url, "status_code": response.status_code})
return failed_requests
# Function to write failed requests to a CSV file
def write_failed_requests(failed_requests):
if not failed_requests:
return
with open(failed_requests_file, 'w', newline='') as csvfile:
fieldnames = ['url', 'status_code']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for request in failed_requests:
writer.writerow(request)
# Function to concatenate all split CSV files into one
def concatenate_csv_files(split_files_dir, output_file):
fieldnames = ['url', 'type']
with open(output_file, 'w', newline='') as outfile:
writer = csv.DictWriter(outfile, fieldnames=fieldnames)
writer.writeheader()
for file in os.listdir(split_files_dir):
if file.endswith('.csv'):
with open(os.path.join(split_files_dir, file), newline='') as infile:
reader = csv.DictReader(infile)
for row in reader:
writer.writerow(row)
# Split the original CSV file into 199 parts
split_csv_file(csv_file_path, split_files_dir)
# Get the list of split CSV files
split_files = [os.path.join(split_files_dir, file) for file in os.listdir(split_files_dir) if file.endswith('.csv')]
# Execute the requests concurrently with 199 threads
all_failed_requests = []
with concurrent.futures.ThreadPoolExecutor(max_workers=199) as executor:
futures = [executor.submit(process_csv_file, split_file) for split_file in split_files]
for future in concurrent.futures.as_completed(futures):
all_failed_requests.extend(future.result())
# Write all failed requests to a file
write_failed_requests(all_failed_requests)
# Concatenate all split CSV files into one final file
concatenate_csv_files(split_files_dir, final_concatenated_file)
print("Processing completed.")

651199
dataset/malicious_phish.csv Normal file
View File

File diff suppressed because it is too large Load Diff

40
dataset/map_type.py Normal file
View File

@@ -0,0 +1,40 @@
import pandas as pd
# Load the CSV files
file1 = pd.read_csv('concatenated_split_files1.csv')
file2 = pd.read_csv('_select_from_safeqr_url_url_left_join_safeqr_qr_code_qr_on_qr_id_202408101634.csv')
# Function to strip 'http://' or 'https://' from a URL
def strip_protocol(url):
if isinstance(url, str):
return url.replace('https://', '').replace('http://', '')
return url
# Apply the strip function to both file1 and file2 URLs
file1['url_stripped'] = file1['url'].apply(strip_protocol)
file2['contents_stripped'] = file2['contents'].apply(strip_protocol)
# Create a dictionary from the second file for quick lookup of type and qr_code_id
url_type_qr_dict = dict(zip(file2['contents_stripped'], zip(file2['result_category'], file2['qr_code_id'])))
# Prepare a copy of file2 to modify without affecting the original
file2_copy = file2.copy()
# Fill in the result_category in file2_copy
file2_copy['result_category'] = file2_copy['contents_stripped'].map(lambda x: url_type_qr_dict[x][0] if x in url_type_qr_dict else None)
# Drop the id and stripped columns in file2_copy
file2_copy = file2_copy.drop(columns=['id', 'contents_stripped'])
# Prepare a copy of file1 to modify without affecting the original
file1_copy = file1.copy()
# Fill in the qr_code_id in file1_copy based on the match from file2
file1_copy['qr_code_id'] = file1_copy['url_stripped'].map(lambda x: url_type_qr_dict[x][1] if x in url_type_qr_dict else None)
# Drop the stripped column in file1_copy
file1_copy = file1_copy.drop(columns=['url_stripped'])
# Save the updated copies to new CSV files
file1_copy.to_csv('file1_updated.csv', index=False)
file2_copy.to_csv('db_updated.csv', index=False)

31138
dataset/ssl_error.csv Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
dataset/url_db_cleaned.csv.zip Normal file
View File

Binary file not shown.

134
gmail-json/test.json Normal file
View File

@@ -0,0 +1,134 @@
{
"emails": [
{
"snippet": "",
"threadId": "190e95e29f2334d8",
"labelIds": [
"IMPORTANT",
"SENT",
"INBOX"
],
"payload": {
"parts": [
{
"parts": [
{
"headers": [
{
"name": "Content-Type",
"value": "text/plain; charset=\"UTF-8\""
}
],
"filename": "",
"mimeType": "text/plain",
"body": {
"data": "DQo=",
"size": 2
},
"partId": "0.0"
},
{
"headers": [
{
"name": "Content-Type",
"value": "text/html; charset=\"UTF-8\""
}
],
"filename": "",
"mimeType": "text/html",
"body": {
"data": "PGRpdj48aW1nIHNyYz0iY2lkOjE5MGU5NWYzZDAwOTZmZjQ5YjQxIiBzdHlsZT0ibWF4LXdpZHRoOiAxMDAlOyI-PC9kaXY-DQo=",
"size": 74
},
"partId": "0.1"
}
],
"headers": [
{
"name": "Content-Type",
"value": "multipart/alternative; boundary=\"0000000000006b7c83061e0f9c47\""
}
],
"filename": "",
"mimeType": "multipart/alternative",
"body": {
"size": 0
},
"partId": "0"
},
{
"headers": [
{
"name": "Content-Type",
"value": "image/png; name=\"6E0889F8-6DAD-417C-A1AF-756DBC1138AC.png\""
},
{
"name": "Content-Disposition",
"value": "attachment; filename=\"6E0889F8-6DAD-417C-A1AF-756DBC1138AC.png\""
},
{
"name": "Content-Transfer-Encoding",
"value": "base64"
},
{
"name": "X-Attachment-Id",
"value": "190e95f3d0096ff49b41"
},
{
"name": "Content-ID",
"value": "<190e95f3d0096ff49b41>"
}
],
"filename": "6E0889F8-6DAD-417C-A1AF-756DBC1138AC.png",
"mimeType": "image/png",
"body": {
"size": 579220,
"attachmentId": "ANGjdJ-HCTgEgqdJXIljB63MrOfI7vChyoOoesaB3u3MGzB038Gv47OSpXQ8DB8l4IpQ8uowl7BUys7Q2dQDv46AylseBwXr4tKmqCJ9DSOFcQnYqOUqN-4t_tlQf7jpI5IoYgpRkqCk6Hr_lihjzfyxSqRGVIgVHJgUvCEJT6jgBEBDDEHqFI8y2JcXyYEuzE51_BCUWTQOUcu6PvVzJSCSvKoLc3zzwYfsMHXr4UX1lSIHAKxDTvfYj-m0KzfPucrcl0ywLD5CvgIJUGsE1X6tbgRRfoHfwL9wdSQHsedre0t_sXNlY592jcy9pUs"
},
"partId": "1"
}
],
"headers": [
{
"name": "MIME-Version",
"value": "1.0"
},
{
"name": "Date",
"value": "Thu, 25 Jul 2024 18:10:57 +0800"
},
{
"name": "Message-ID",
"value": "<CAOhDyE09kVfVpYTOwm6cwir9tLaYvs9JM-4ZJsNQpk57uOOd_w@mail.gmail.com>"
},
{
"name": "Subject",
"value": "Test Test"
},
{
"name": "From",
"value": "SafeQR <safeqr.fyp.24.s2.43p@gmail.com>"
},
{
"name": "To",
"value": "SafeQR <safeqr.fyp.24.s2.43p@gmail.com>"
},
{
"name": "Content-Type",
"value": "multipart/related; boundary=\"0000000000006b7c84061e0f9c48\""
}
],
"filename": "",
"mimeType": "multipart/related",
"body": {
"size": 0
},
"partId": ""
},
"historyId": 27133,
"id": "190e95f5626b2a39",
"sizeEstimate": 793666,
"internalDate": 1721902257000
}
]
}

14257
gmail-json/test_all.json Normal file
View File

File diff suppressed because one or more lines are too long

11
gmail-json/test_tree.json Normal file
View File

@@ -0,0 +1,11 @@
{
"cartoon": {
"0": {
"more than 1 person": {
"0": "no",
"1": { "winter": { "0": "no", "1": "yes" } }
}
},
"1": "no"
}
}

2028
gmail-json/with_qr.json Normal file
View File

File diff suppressed because one or more lines are too long

69
serverless/safeqr-signup-post-confirmation-lambda.py Normal file
View File

@@ -0,0 +1,69 @@
import os
import pg8000
def lambda_handler(event, context):
print(f"Event: {event}")
# Extract user attributes from the Cognito event
user_attributes = event['request']['userAttributes']
# Extract specific attributes (modify as needed)
user_id = user_attributes['sub']
email = user_attributes['email']
name = user_attributes.get('name', 'Default User')
# Extract providerName from identities
identities = user_attributes.get('identities', [])
provider_name = identities[0]['providerName'] if identities else 'DefaultApp'
print(f"providerName: {provider_name}")
# Database connection parameters
db_host = os.environ['RDS_HOST']
db_name = os.environ['RDS_DB_NAME']
db_user = os.environ['RDS_USER']
db_password = os.environ['RDS_PASSWORD']
db_port = 5432
# Connect to the database
try:
conn = pg8000.connect(
host=db_host,
port=db_port,
database=db_name,
user=db_user,
password=db_password
)
# Create a cursor
cur = conn.cursor()
print("Connection to db success!")
# SQL query to insert user details
insert_query = """
INSERT INTO safeqr."user" (id, email, name, roles, source)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (id) DO UPDATE
SET email = EXCLUDED.email, name = EXCLUDED.name, source = EXCLUDED.source;
"""
# Execute the query
cur.execute(insert_query, (user_id, email, name, ['appuser'], provider_name))
# Commit the transaction
conn.commit()
print(f"User {user_id} inserted/updated successfully")
except Exception as e:
print(f"Database error: {str(e)}")
raise e
finally:
# Close the cursor and connection
if 'cur' in locals():
cur.close()
if 'conn' in locals():
conn.close()
# Return the event object back to Cognito
return event

291
sql/Create_all_tables.sql Normal file
View File

@@ -0,0 +1,291 @@
-- Create schema if it doesn't exist
CREATE SCHEMA IF NOT EXISTS safeqr;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Create Enum for scan_history status
CREATE TYPE scan_status AS ENUM ('ACTIVE', 'INACTIVE');
-- safeqr."user" definition
-- Drop table
-- DROP TABLE safeqr."user";
CREATE TABLE safeqr."user" (
id varchar(255) NOT NULL,
"name" varchar(255) NULL,
email varchar(255) NULL,
"source" varchar(255) NULL,
date_created timestamptz DEFAULT now() NULL,
date_updated timestamptz DEFAULT now() NULL,
roles _text NULL,
status varchar(255) DEFAULT 'ACTIVE'::character varying NULL,
gmail_history_id int8 DEFAULT '-1'::integer NULL,
CONSTRAINT user_pkey PRIMARY KEY (id)
);
-- safeqr.qr_code_types definition
-- Drop table
-- DROP TABLE safeqr.qr_code_types;
CREATE TABLE safeqr.qr_code_types (
id bigserial NOT NULL,
"type" varchar(255) NOT NULL,
description varchar(255) NULL,
prefix varchar(255) NULL,
table_name varchar(255) NULL,
CONSTRAINT qr_code_types_pkey PRIMARY KEY (id)
);
-- safeqr.qr_code definition
-- Drop table
-- DROP TABLE safeqr.qr_code;
CREATE TABLE safeqr.qr_code (
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
qr_code_type_id int8 NULL,
user_id varchar(255) NULL,
contents text NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NULL,
result_category text NULL,
CONSTRAINT qr_code_pkey PRIMARY KEY (id),
CONSTRAINT qr_code_qr_code_type_id_fkey FOREIGN KEY (qr_code_type_id) REFERENCES safeqr.qr_code_types(id),
CONSTRAINT qr_code_user_id_fkey FOREIGN KEY (user_id) REFERENCES safeqr."user"(id)
);
-- safeqr.scan_history definition
-- Drop table
-- DROP TABLE safeqr.scan_history;
CREATE TABLE safeqr.scan_history (
id bigserial NOT NULL,
qr_code_id uuid NULL,
user_id varchar(255) NULL,
status varchar(255) DEFAULT 'ACTIVE'::safeqr.scan_status NULL,
date_created timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
date_updated timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
bookmarked bool DEFAULT false NULL,
CONSTRAINT scan_history_pkey PRIMARY KEY (id),
CONSTRAINT scan_history_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
);
-- safeqr.scan_bookmark definition
-- Drop table
-- DROP TABLE safeqr.scan_bookmark;
CREATE TABLE safeqr.scan_bookmark (
id bigserial NOT NULL,
qr_code_id uuid NULL,
user_id varchar(255) NULL,
status varchar(255) DEFAULT 'ACTIVE'::safeqr.bookmark_status NULL,
date_created timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
date_updated timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
CONSTRAINT scan_bookmark_pkey PRIMARY KEY (id),
CONSTRAINT scan_bookmark_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
);
-- safeqr.url definition
-- Drop table
-- DROP TABLE safeqr.url;
CREATE TABLE safeqr.url (
id uuid DEFAULT uuid_generate_v4() NOT NULL,
qr_code_id uuid NULL,
"domain" text NULL,
subdomain text NULL,
top_level_domain text NULL,
query text NULL,
fragment text NULL,
redirect int4 DEFAULT 0 NULL,
"path" text NULL,
redirect_chain _text NULL,
hsts_header _text NULL,
ssl_stripping _bool NULL,
ssl_error text NULL,
dns_error text NULL,
hostname_embedding int4 NULL,
javascript_check text NULL,
shortening_service text NULL,
has_ip_address text NULL,
tracking_descriptions _text NULL,
url_encoding text NULL,
classifications text NULL,
has_executable text NULL,
CONSTRAINT url_pkey PRIMARY KEY (id),
CONSTRAINT url_qr_code_id_fkey FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id)
);
-- safeqr."text" definition
-- Drop table
-- DROP TABLE safeqr."text";
CREATE TABLE safeqr."text" (
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
qr_code_id uuid NULL,
"text" varchar(2048) NULL,
CONSTRAINT text_pkey PRIMARY KEY (id),
CONSTRAINT text_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
);
-- safeqr.phone definition
-- Drop table
-- DROP TABLE safeqr.phone;
CREATE TABLE safeqr.phone (
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
qr_code_id uuid NULL,
phone text NULL,
CONSTRAINT phone_pkey PRIMARY KEY (id),
CONSTRAINT phone_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
);
-- safeqr.sms definition
-- Drop table
-- DROP TABLE safeqr.sms;
CREATE TABLE safeqr.sms (
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
qr_code_id uuid NULL,
phone text NULL,
message text NULL,
CONSTRAINT sms_pkey PRIMARY KEY (id),
CONSTRAINT sms_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id)
);
-- Create Email table
CREATE TABLE safeqr.email (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
qr_code_id UUID REFERENCES safeqr.qr_code(id),
email VARCHAR(200),
title VARCHAR(200),
message VARCHAR(7089)
);
-- Create GeoLocation table
CREATE TABLE safeqr.geolocation (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
qr_code_id UUID REFERENCES safeqr.qr_code(id),
latitude DOUBLE PRECISION,
longitude DOUBLE PRECISION
);
-- Create Wifi table
CREATE TABLE safeqr.wifi (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
qr_code_id UUID REFERENCES safeqr.qr_code(id),
ssid VARCHAR(200),
password VARCHAR(200),
encryption VARCHAR(200),
hidden BOOLEAN
);
-- Create Bitcoin table
CREATE TABLE safeqr.bitcoin (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
qr_code_id UUID REFERENCES safeqr.qr_code(id),
address VARCHAR(200),
amount BIGINT,
message VARCHAR(250)
);
-- Create VCard table
CREATE TABLE safeqr.vcard (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
qr_code_id UUID REFERENCES safeqr.qr_code(id),
version VARCHAR(50),
name VARCHAR(250),
organisation VARCHAR(250),
title VARCHAR(50),
address VARCHAR(250),
tel_work VARCHAR(250),
tel_cell VARCHAR(250),
email VARCHAR(250),
url VARCHAR(2048)
);
-- safeqr.gmail_emails definition
-- Drop table
-- DROP TABLE safeqr.gmail_emails;
CREATE TABLE safeqr.gmail_emails (
user_id varchar NOT NULL,
subject text NULL,
date_received timestamptz NULL,
date_created timestamptz DEFAULT CURRENT_TIMESTAMP NULL,
message_id varchar NOT NULL,
history_id int8 NULL,
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
CONSTRAINT gmail_pk PRIMARY KEY (id),
CONSTRAINT gmail_unique UNIQUE (user_id, message_id),
CONSTRAINT gmail_user_fk FOREIGN KEY (user_id) REFERENCES safeqr."user"(id) ON DELETE CASCADE ON UPDATE CASCADE
);
-- safeqr.url definition
-- Drop table
-- DROP TABLE safeqr.url;
CREATE TABLE safeqr.url (
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
qr_code_id uuid NULL,
"domain" text NULL,
subdomain text NULL,
top_level_domain text NULL,
query text NULL,
fragment text NULL,
redirect int4 DEFAULT 0 NULL,
"path" text NULL,
redirect_chain _text NULL,
hsts_header _text NULL,
ssl_stripping _bool NULL,
ssl_error text NULL,
dns_error text NULL,
hostname_embedding int4 NULL,
javascript_check text NULL,
shortening_service text NULL,
has_ip_address text NULL,
tracking_descriptions _text NULL,
CONSTRAINT url_pkey PRIMARY KEY (id),
CONSTRAINT url_qr_code_id_fkey FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id)
);
-- safeqr.gmail_cid definition
-- Drop table
-- DROP TABLE safeqr.gmail_cid;
CREATE TABLE safeqr.gmail_cid (
gmail_id uuid NOT NULL,
cid varchar NOT NULL,
attachment_id text NOT NULL,
decoded_content text NOT NULL,
qr_code_id uuid NOT NULL,
id uuid DEFAULT safeqr.uuid_generate_v4() NOT NULL,
CONSTRAINT gmail_cid_pk PRIMARY KEY (id),
CONSTRAINT gmail_cid_unique UNIQUE (gmail_id, cid, decoded_content),
CONSTRAINT gmail_cid_gmail_emails_fk FOREIGN KEY (gmail_id) REFERENCES safeqr.gmail_emails(id) ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT gmail_cid_qr_code_fk FOREIGN KEY (qr_code_id) REFERENCES safeqr.qr_code(id) ON DELETE CASCADE ON UPDATE CASCADE
);

16
sql/Drop_all_tables.sql Normal file
View File

@@ -0,0 +1,16 @@
-- Drop the tables with CASCADE to handle dependencies
DROP TABLE IF EXISTS safeqr.vcard CASCADE;
DROP TABLE IF EXISTS safeqr.bitcoin CASCADE;
DROP TABLE IF EXISTS safeqr.wifi CASCADE;
DROP TABLE IF EXISTS safeqr.geolocation CASCADE;
DROP TABLE IF EXISTS safeqr.email CASCADE;
DROP TABLE IF EXISTS safeqr.sms CASCADE;
DROP TABLE IF EXISTS safeqr.phone CASCADE;
DROP TABLE IF EXISTS safeqr.text CASCADE;
DROP TABLE IF EXISTS safeqr.url CASCADE;
DROP TABLE IF EXISTS safeqr.scan_bookmark CASCADE;
DROP TABLE IF EXISTS safeqr.scan_history CASCADE;
DROP TABLE IF EXISTS safeqr.qr_code CASCADE;
DROP TABLE IF EXISTS safeqr.qr_code_types CASCADE;
DROP TABLE IF EXISTS safeqr."user" CASCADE;

97
sql/Dummy_data.sql Normal file
View File

@@ -0,0 +1,97 @@
INSERT INTO safeqr."user" (
id,
cognitoid,
firstname,
lastname,
email,
source,
password,
salt,
cognito_id,
first_name,
last_name
) VALUES (
'test-unique-id',
'test-cognito-id',
'Test',
'User',
'test.user@example.com',
'test-source',
'test-password-hash',
'test-salt',
NULL,
NULL,
NULL
);
-- Insert into QR_Code_Types table
-- Insert additional QR code types into the qr_code_types table
INSERT INTO safeqr.qr_code_types (type, description, prefix, table_name) VALUES
('URL', 'Uniform Resource Locator', 'http://', 'url'),
('URL', 'Uniform Resource Locator', 'https://', 'url'),
('EMAIL', 'Email Address', 'mailto:', 'email'),
('PHONE', 'Phone Number', 'tel:', 'phone'),
('SMS', 'Short Message Service', 'smsto:', 'sms'),
('GEOLOCATION', 'Geographic Location', 'geo:', 'geolocation'),
('WIFI', 'Wireless Network Configuration', 'WIFI:', 'wifi'),
('BITCOIN', 'Cryptocurrency Address', 'bitcoin:', 'bitcoin'),
('VCARD', 'Virtual Contact File', 'BEGIN:VCARD', 'vcard'),
('TEXT', 'Text', '', 'text');
-- Insert into QR_Code table
INSERT INTO safeqr.qr_code (id, qr_code_type_id, user_id, contents, created_at) VALUES
(gen_random_uuid(), 1, 'test-unique-id', 'http://example.com', CURRENT_TIMESTAMP),
(gen_random_uuid(), 2, NULL, 'https://example.com', CURRENT_TIMESTAMP),
(gen_random_uuid(), 3, 'test-unique-id', 'mailto:test@example.com', CURRENT_TIMESTAMP);
-- Insert into Scan_History table
INSERT INTO safeqr.scan_history (qr_code_id, user_id) VALUES
((SELECT id FROM safeqr.qr_code LIMIT 1), 'test-unique-id'),
((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 1), 'test-unique-id'),
((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 2), 'test-unique-id');
-- Insert into Scan_Bookmark table
INSERT INTO safeqr.scan_bookmark (qr_code_id, user_id, active) VALUES
((SELECT id FROM safeqr.qr_code LIMIT 1), 'test-unique-id', 'ACTIVE'),
((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 1), 'test-unique-id', 'ACTIVE'),
((SELECT id FROM safeqr.qr_code LIMIT 1 OFFSET 2), 'test-unique-id', 'DELETED');
-- Insert into URL table
INSERT INTO safeqr.url (id, qr_code_id, url, query, fragment) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents LIKE 'http://%' LIMIT 1), 'http://example.com', 'key=value', 'fragment'),
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents LIKE 'https://%' LIMIT 1), 'https://example.com', 'key=value', 'fragment');
-- Insert into Text table
INSERT INTO safeqr.text (id, qr_code_id, text) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents = 'Some text' LIMIT 1), 'Some example text');
-- Insert into Phone table
INSERT INTO safeqr.phone (id, qr_code_id, phone) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '+1234567890');
-- Insert into SMS table
INSERT INTO safeqr.sms (id, qr_code_id, phone, message) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '+1234567890', 'This is a test SMS message.');
-- Insert into Email table
INSERT INTO safeqr.email (id, qr_code_id, email, title, message) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code WHERE contents LIKE 'mailto:%' LIMIT 1), 'test@example.com', 'Test Email', 'This is a test email body.');
-- Insert into GeoLocation table
INSERT INTO safeqr.geolocation (id, qr_code_id, latitude, longitude) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), 37.7749, -122.4194);
-- Insert into Wifi table
INSERT INTO safeqr.wifi (id, qr_code_id, ssid, password, encryption, hidden) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), 'TestSSID', 'password123', 'WPA2', FALSE);
-- Insert into Bitcoin table
INSERT INTO safeqr.bitcoin (id, qr_code_id, address, amount, message) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', 5000000000, 'Test Bitcoin transaction');
-- Insert into VCard table
INSERT INTO safeqr.vcard (id, qr_code_id, version, name, organisation, title, address, tel_work, tel_cell, email, url) VALUES
(gen_random_uuid(), (SELECT id FROM safeqr.qr_code LIMIT 1), '4.0', 'John Doe', 'Example Corp', 'Developer', '1234 Elm St, Springfield, IL', '+1234567890', '+0987654321', 'johndoe@example.com', 'http://example.com');

11
sql/qr_code_types_202408032138.sql Normal file
View File

@@ -0,0 +1,11 @@
INSERT INTO safeqr.qr_code_types ("type",description,prefix,table_name) VALUES
('URL','Uniform Resource Locator','http://','url'),
('EMAIL','Email Address','mailto:','email'),
('PHONE','Phone Number','tel:','phone'),
('SMS','Short Message Service','smsto:','sms'),
('GEOLOCATION','Geographic Location','geo:','geolocation'),
('WIFI','Wireless Network Configuration','WIFI:','wifi'),
('BITCOIN','Cryptocurrency Address','bitcoin:','bitcoin'),
('VCARD','Virtual Contact File','BEGIN:VCARD','vcard'),
('URL','Secure Uniform Resource Locator','https://','url'),
('TEXT','Text','','text');