From 1d1ffcf5dc7d6b69cdc3c15d7652d13fe6f55e66 Mon Sep 17 00:00:00 2001
From: heyethereum <heyethereum@gmail.com>
Date: Mon, 5 Aug 2024 08:16:53 +0800
Subject: [PATCH 1/5] added error checking for connection error

---
 .../safeqr/app/qrcode/entity/URLEntity.java   |  6 +++++
 .../service/URLVerificationService.java       | 23 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
index 476f223..a75ae88 100644
--- a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
+++ b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
@@ -58,4 +58,10 @@ public class URLEntity {
     @Type(ListArrayType.class)
     @Column(name = "redirect_chain", columnDefinition = "text[]")
     private List<String> redirectChain;
+
+    @Column(name = "dns_error")
+    private String dnsError;
+
+    @Column(name="certificate_subject_mismatch")
+    private String certificateSubjectMismatch;
 }
diff --git a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
index 5649f1a..3cdf75e 100644
--- a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
+++ b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
@@ -2,7 +2,6 @@ package com.safeqr.app.qrcode.service;
 
 import static com.safeqr.app.constants.CommonConstants.*;
 
-import com.safeqr.app.exceptions.ResourceNotFoundExceptions;
 import com.safeqr.app.qrcode.dto.request.QRCodePayload;
 import com.safeqr.app.qrcode.dto.URLVerificationResponse;
 import com.safeqr.app.qrcode.entity.URLEntity;
@@ -13,6 +12,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLHandshakeException;
 import java.io.IOException;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
@@ -21,6 +21,8 @@ import java.util.*;
 
 @Service
 public class URLVerificationService {
+    private static final int CONNECTION_TIMEOUT_MS = 10000;
+    private static final int READ_TIMEOUT_MS = 10000;
     private static final Logger logger = LoggerFactory.getLogger(URLVerificationService.class);
     private final URLRepository urlRepository;
     @Autowired
@@ -144,6 +146,8 @@ public class URLVerificationService {
                 HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                 connection.setRequestMethod("GET");
                 connection.setInstanceFollowRedirects(false);
+                connection.setConnectTimeout(CONNECTION_TIMEOUT_MS);
+                connection.setReadTimeout(READ_TIMEOUT_MS);
 
                 int responseCode = connection.getResponseCode();
                 redirected = (responseCode >= 300 && responseCode < 400);
@@ -182,6 +186,23 @@ public class URLVerificationService {
             details.setHstsHeader(hstsHeaderList);
         } catch (URISyntaxException e){
             logger.error("Error in breaking down URL: {}", e.getMessage());
+        } catch (SSLHandshakeException e) {
+            logger.error("SSL Handshake Exception: {}", e.getMessage());
+            details.setCertificateSubjectMismatch("SSL Handshake Exception: " + e.getMessage());
+        } catch (SocketTimeoutException e) {
+            logger.error("Connection timed out: {}", e.getMessage());
+            details.setDnsError("Connection timed out: " + e.getMessage());
+        } catch (UnknownHostException e) {
+            logger.error("Unknown Host Exception: {}", e.getMessage());
+            details.setDnsError("Unknown Host Exception: " + e.getMessage());
+        } catch (NoRouteToHostException e) {
+            details.setDnsError("Error: No route to host: " + e.getMessage());
+        } catch (ConnectException e) {
+            details.setDnsError("Error: Connection refused: " + e.getMessage());
+        } catch (SocketException e) {
+            details.setDnsError("Error: Network is unreachable or other socket error: " + e.getMessage());
+        } catch (Exception e) {
+            details.setDnsError("Exception: " + e.getMessage());
         }
     }
     // Function to check if the redirect is from HTTPS to HTTP

From 2746891645049f7a033de453ba8571f833489170 Mon Sep 17 00:00:00 2001
From: ltiongku <alex.b.lim@accenture.com>
Date: Mon, 5 Aug 2024 20:45:03 +0800
Subject: [PATCH 2/5] added javascript check in url and domain embedding checks

---
 .../safeqr/app/qrcode/entity/URLEntity.java   |  15 ++-
 .../service/URLVerificationService.java       | 108 ++++++++++++------
 2 files changed, 81 insertions(+), 42 deletions(-)

diff --git a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
index a75ae88..012625c 100644
--- a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
+++ b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
@@ -11,6 +11,7 @@ import lombok.Builder;
 import org.hibernate.annotations.Type;
 import org.hibernate.annotations.UuidGenerator;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 
@@ -49,18 +50,24 @@ public class URLEntity {
 
     @Type(ListArrayType.class)
     @Column(name = "hsts_header", columnDefinition = "text[]")
-    private List<String> hstsHeader;
+    private List<String> hstsHeader = new ArrayList<>();
 
     @Type(ListArrayType.class)
     @Column(name = "ssl_stripping", columnDefinition = "boolean[]")
-    private List<Boolean> sslStripping;
+    private List<Boolean> sslStripping = new ArrayList<>();
 
     @Type(ListArrayType.class)
     @Column(name = "redirect_chain", columnDefinition = "text[]")
-    private List<String> redirectChain;
+    private List<String> redirectChain = new ArrayList<>();
+
+    @Column(name = "hostname_embedding")
+    private int hostnameEmbedding = 0;
+
+    @Column(name = "javascript_check")
+    private String javascriptCheck = "No Javascript in URL";
 
     @Column(name = "dns_error")
-    private String dnsError;
+    private String dnsError = "No Error Found.";
 
     @Column(name="certificate_subject_mismatch")
     private String certificateSubjectMismatch;
diff --git a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
index 3cdf75e..6f59b97 100644
--- a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
+++ b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
@@ -17,6 +17,9 @@ import java.io.IOException;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 
 @Service
@@ -41,55 +44,84 @@ public class URLVerificationService {
         urlRepository.save(urlEntity);
     }
     // Function to breakdown URL into subdomain, domain, topLevelDomain, query params, fragment
-    public URLEntity breakdownURL(String urlString) throws MalformedURLException {
+    public URLEntity breakdownURL(String urlString) {
         URLEntity urlObj = new URLEntity();
         try {
-            // Ensure the URL is properly encoded
-            String encodedUrl = encodeUrl(urlString);
-            URI uri = new URI(encodedUrl);
-            URL url = uri.toURL();
-
+            URL url = new URI(encodeUrl(urlString)).toURL();
             String host = url.getHost();
-            // split host into subdomain, domain, topLevelDomain
-            String[] hostParts = host.split("\\.");
-            String subdomain = "";
 
-            if (hostParts.length >= 2) {
-                // set topLevelDomain to the last part of the host
-                urlObj.setTopLevelDomain(hostParts[hostParts.length - 1]);
-                // set domain to the second last part of the host
-                urlObj.setDomain(hostParts[hostParts.length - 2]);
-                // set subdomain to the first part of the host
-                if (hostParts.length > 2) {
-                    subdomain = String.join(".", java.util.Arrays.copyOfRange(hostParts, 0, hostParts.length - 2));
-                }
-            }
-            // set subdomain to URL host
-            urlObj.setSubdomain(subdomain);
+            urlObj.setHostnameEmbedding(checkDeceptiveUrl(url));
+            urlObj.setJavascriptCheck(checkForJavascriptCode(urlString));
 
-            String path = url.getPath();
-            //set path to URL path if it's not empty, otherwise set it to root path
-            urlObj.setPath(path.isEmpty() ? "/" : path);
+            populateHostDetails(host, urlObj);
 
-            String query = url.getQuery();
-            Map<String, String> queryParams = new HashMap<>();
-            if (query != null) {
-                // split query params into key value pairs
-                for (String param : query.split("&")) {
-                    String[] pair = param.split("=");
-                    queryParams.put(pair[0], pair.length > 1 ? pair[1] : "");
-                }
-                logger.info("queryParams: {}", queryParams);
-            }
-            // set query params to URL query
-            urlObj.setQuery(queryParams.toString());
-            // set fragment to URL ref
+            urlObj.setPath(Optional.ofNullable(url.getPath()).filter(p -> !p.isEmpty()).orElse("/"));
+            urlObj.setQuery(parseQueryParams(url.getQuery()));
             urlObj.setFragment(Optional.ofNullable(url.getRef()).orElse(""));
-        } catch (URISyntaxException | MalformedURLException e) {
+
+        } catch (Exception e) {
             logger.error("Error in breaking down URL: {}", e.getMessage());
         }
         return urlObj;
     }
+
+    private void populateHostDetails(String host, URLEntity urlObj) {
+        String[] hostParts = host.split("\\.");
+        int length = hostParts.length;
+
+        if (length >= 2) {
+            urlObj.setTopLevelDomain(hostParts[length - 1]);
+            urlObj.setDomain(hostParts[length - 2]);
+            urlObj.setSubdomain(length > 2 ? String.join(".", Arrays.copyOfRange(hostParts, 0, length - 2)) : "");
+        }
+    }
+
+    private int checkDeceptiveUrl(URL url) {
+        String[] parts = url.getHost().split("\\.");
+        if (parts.length < 3) return 0;
+
+        Set<String> commonTlds = new HashSet<>(Arrays.asList("com", "org", "net", "edu", "gov"));
+
+        for (int i = parts.length - 2; i >= 1; i--) {
+            if (commonTlds.contains(parts[i]) && !commonTlds.contains(parts[i - 1]) && i != parts.length - 2) {
+                logger.warn("Potentially deceptive URL detected: {} (Suspicious domain: {}.{})",
+                        url, parts[i - 1], parts[i]);
+                return 1;
+            }
+        }
+        return 0;
+    }
+
+    private String checkForJavascriptCode(String url) {
+        // Patterns to detect 'javascript:', '<script>', and 'on*=' attributes
+        List<Pattern> maliciousPatterns = Arrays.asList(
+                Pattern.compile("javascript:", Pattern.CASE_INSENSITIVE),
+                Pattern.compile("<\\s*script", Pattern.CASE_INSENSITIVE),
+                Pattern.compile("on\\w*\\s*=", Pattern.CASE_INSENSITIVE)
+        );
+
+        // Check for any malicious pattern in the URL
+        for (Pattern pattern : maliciousPatterns) {
+            Matcher matcher = pattern.matcher(url);
+            if (matcher.find()) {
+                return "Javascript found in URL!";
+            }
+        }
+        return "No Javascript in URL";
+    }
+    private String parseQueryParams(String query) {
+        if (query == null) return "{}";
+
+        Map<String, String> queryParams = Arrays.stream(query.split("&"))
+                .map(param -> param.split("="))
+                .collect(Collectors.toMap(
+                        pair -> pair[0],
+                        pair -> pair.length > 1 ? pair[1] : "",
+                        (oldValue, newValue) -> oldValue, HashMap::new));
+
+        return queryParams.toString();
+    }
+
     private String encodeUrl(String urlString) throws MalformedURLException {
         try {
             URL url = new URL(urlString);

From 02085b50b9554ed986d518bec0cdf7feea2dbea6 Mon Sep 17 00:00:00 2001
From: heyethereum <heyethereum@gmail.com>
Date: Tue, 6 Aug 2024 00:41:17 +0800
Subject: [PATCH 3/5] added tracking function, check has ip in url, url
 shortener check

---
 .../safeqr/app/qrcode/entity/URLEntity.java   |  18 +++-
 .../service/URLVerificationService.java       | 100 ++++++++++++++++--
 2 files changed, 108 insertions(+), 10 deletions(-)

diff --git a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
index 012625c..df7192d 100644
--- a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
+++ b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
@@ -64,11 +64,21 @@ public class URLEntity {
     private int hostnameEmbedding = 0;
 
     @Column(name = "javascript_check")
-    private String javascriptCheck = "No Javascript in URL";
+    private String javascriptCheck = "";
+
+    @Column(name = "shortening_service")
+    private String shorteningService = "";
+
+    @Column(name = "has_ip_address")
+    private String hasIpAddress = "";
+
+    @Type(ListArrayType.class)
+    @Column(name = "tracking_descriptions", columnDefinition = "text[]")
+    private List<String> trackingDescriptions = new ArrayList<>();
 
     @Column(name = "dns_error")
-    private String dnsError = "No Error Found.";
+    private String dnsError = "";
 
-    @Column(name="certificate_subject_mismatch")
-    private String certificateSubjectMismatch;
+    @Column(name="ssl_error")
+    private String sslError = "";
 }
diff --git a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
index 6f59b97..5650a9f 100644
--- a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
+++ b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
@@ -33,6 +33,29 @@ public class URLVerificationService {
         this.urlRepository = urlRepository;
     }
 
+    // Regular expression pattern for shortening services
+    private static final String SHORTENING_PATTERN =
+            "bit\\.ly|goo\\.gl|shorte\\.st|go2l\\.ink|x\\.co|ow\\.ly|t\\.co|tinyurl|tr\\.im|is\\.gd|cli\\.gs|" +
+                    "yfrog\\.com|migre\\.me|ff\\.im|tiny\\.cc|url4\\.eu|twit\\.ac|su\\.pr|twurl\\.nl|snipurl\\.com|" +
+                    "short\\.to|BudURL\\.com|ping\\.fm|post\\.ly|Just\\.as|bkite\\.com|snipr\\.com|fic\\.kr|loopt\\.us|" +
+                    "doiop\\.com|short\\.ie|kl\\.am|wp\\.me|rubyurl\\.com|om\\.ly|to\\.ly|bit\\.do|t\\.co|lnkd\\.in|" +
+                    "db\\.tt|qr\\.ae|adf\\.ly|goo\\.gl|bitly\\.com|cur\\.lv|tinyurl\\.com|ow\\.ly|bit\\.ly|ity\\.im|" +
+                    "q\\.gs|is\\.gd|po\\.st|bc\\.vc|twitthis\\.com|u\\.to|j\\.mp|buzurl\\.com|cutt\\.us|u\\.bb|yourls\\.org|" +
+                    "x\\.co|prettylinkpro\\.com|scrnch\\.me|filoops\\.info|vzturl\\.com|qr\\.net|1url\\.com|tweez\\.me|v\\.gd|" +
+                    "tr\\.im|link\\.zip\\.net";
+
+    // Regular expression pattern to match various IP address formats
+    private static final String IP_PATTERN =
+            "(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+                    "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\/)|" +
+                    "(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+                    "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\/)|" +
+                    "((0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\/)" +
+                    "(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4}|" +
+                    "([0-9]+(?:\\.[0-9]+){3}:[0-9]+)|" +
+                    "((?:(?:\\d|[01]?\\d\\d|2[0-4]\\d|25[0-5])\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d|\\d)(?:\\/\\d{1,2})?)";
+
+
     public URLEntity getURLEntityByQRCodeId(UUID qrCodeId) {
         logger.info("qrCodeId retrieving: {}", qrCodeId);
 //        return urlRepository.findByQrCodeId(qrCodeId)
@@ -50,15 +73,26 @@ public class URLVerificationService {
             URL url = new URI(encodeUrl(urlString)).toURL();
             String host = url.getHost();
 
+            // Check for deceptive URL
             urlObj.setHostnameEmbedding(checkDeceptiveUrl(url));
+
+            // Check for Javascript code in url
             urlObj.setJavascriptCheck(checkForJavascriptCode(urlString));
 
+            // Check for url shortener
+            urlObj.setShorteningService(hasShorteningService(urlString));
+
+            // Check for IP address
+            urlObj.setHasIpAddress(hasIPAddress(urlString));
+
             populateHostDetails(host, urlObj);
 
             urlObj.setPath(Optional.ofNullable(url.getPath()).filter(p -> !p.isEmpty()).orElse("/"));
             urlObj.setQuery(parseQueryParams(url.getQuery()));
             urlObj.setFragment(Optional.ofNullable(url.getRef()).orElse(""));
 
+            // Check for tracking parameters
+            urlObj.setTrackingDescriptions(getTrackingDescriptions(url.getQuery()));
         } catch (Exception e) {
             logger.error("Error in breaking down URL: {}", e.getMessage());
         }
@@ -75,6 +109,46 @@ public class URLVerificationService {
             urlObj.setSubdomain(length > 2 ? String.join(".", Arrays.copyOfRange(hostParts, 0, length - 2)) : "");
         }
     }
+    // List of common tracking parameters with their descriptions
+    private static final Map<String, String> TRACKING_DESCRIPTIONS = Map.ofEntries(
+            Map.entry("utm_source", "Campaign Source: Identifies which site sent the traffic."),
+            Map.entry("utm_medium", "Campaign Medium: Identifies what type of link was used."),
+            Map.entry("utm_campaign", "Campaign Name: Identifies a specific product promotion or campaign."),
+            Map.entry("utm_term", "Campaign Term: Identifies search terms."),
+            Map.entry("utm_content", "Campaign Content: Differentiates similar content or links within the same ad."),
+            Map.entry("gclid", "Google Click Identifier: Used by Google Ads to track clicks."),
+            Map.entry("fbclid", "Facebook Click Identifier: Used by Facebook to track clicks."),
+            Map.entry("tracking_id", "Tracking ID: General identifier for tracking purposes."),
+            Map.entry("affiliate_id", "Affiliate ID: Identifies traffic from affiliates."),
+            Map.entry("ref", "Referrer: Identifies the referrer site."),
+            Map.entry("referrer", "Referrer: Identifies the referrer site.")
+    );
+
+    // Regex pattern to capture key-value pairs in the query string
+    private static final Pattern PARAM_PATTERN = Pattern.compile(
+            "(?<key>[^=&]+)=(?<value>[^&]+)",
+            Pattern.CASE_INSENSITIVE
+    );
+
+    // Static method to detect and return tracking parameter descriptions in a URL
+    private List<String> getTrackingDescriptions(String query) {
+        if (query == null || query.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        Matcher matcher = PARAM_PATTERN.matcher(query);
+        List<String> foundDescriptions = new ArrayList<>();
+
+        while (matcher.find()) {
+            String key = matcher.group("key").toLowerCase();
+            String value = URLDecoder.decode(matcher.group("value"), StandardCharsets.UTF_8);
+            if (TRACKING_DESCRIPTIONS.containsKey(key)) {
+                foundDescriptions.add(TRACKING_DESCRIPTIONS.get(key) + ": " + value);
+            }
+        }
+
+        return foundDescriptions;
+    }
 
     private int checkDeceptiveUrl(URL url) {
         String[] parts = url.getHost().split("\\.");
@@ -104,10 +178,24 @@ public class URLVerificationService {
         for (Pattern pattern : maliciousPatterns) {
             Matcher matcher = pattern.matcher(url);
             if (matcher.find()) {
-                return "Javascript found in URL!";
+                return "Javascript found in URL.";
             }
         }
-        return "No Javascript in URL";
+        return "";
+    }
+
+    // Function to detect if the URL uses a shortening service
+    private String hasShorteningService(String url) {
+        Pattern pattern = Pattern.compile(SHORTENING_PATTERN, Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(url);
+        return matcher.find() ? "Yes" : "";
+    }
+
+    // Function to detect if the URL has an IP address
+    private static String hasIPAddress(String url) {
+        Pattern pattern = Pattern.compile(IP_PATTERN, Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(url);
+        return matcher.find() ? "URL contains IP address." : "";
     }
     private String parseQueryParams(String query) {
         if (query == null) return "{}";
@@ -220,7 +308,7 @@ public class URLVerificationService {
             logger.error("Error in breaking down URL: {}", e.getMessage());
         } catch (SSLHandshakeException e) {
             logger.error("SSL Handshake Exception: {}", e.getMessage());
-            details.setCertificateSubjectMismatch("SSL Handshake Exception: " + e.getMessage());
+            details.setSslError("SSL Handshake Exception: " + e.getMessage());
         } catch (SocketTimeoutException e) {
             logger.error("Connection timed out: {}", e.getMessage());
             details.setDnsError("Connection timed out: " + e.getMessage());
@@ -228,11 +316,11 @@ public class URLVerificationService {
             logger.error("Unknown Host Exception: {}", e.getMessage());
             details.setDnsError("Unknown Host Exception: " + e.getMessage());
         } catch (NoRouteToHostException e) {
-            details.setDnsError("Error: No route to host: " + e.getMessage());
+            details.setDnsError("Error: " + e.getMessage());
         } catch (ConnectException e) {
-            details.setDnsError("Error: Connection refused: " + e.getMessage());
+            details.setDnsError("Connection Error: " + e.getMessage());
         } catch (SocketException e) {
-            details.setDnsError("Error: Network is unreachable or other socket error: " + e.getMessage());
+            details.setDnsError("Socket Error: " + e.getMessage());
         } catch (Exception e) {
             details.setDnsError("Exception: " + e.getMessage());
         }

From 3eb53c6ccd34c1665e4c638e5ddf9eb3bb461331 Mon Sep 17 00:00:00 2001
From: heyethereum <heyethereum@gmail.com>
Date: Tue, 6 Aug 2024 08:00:03 +0800
Subject: [PATCH 4/5] added url encoding check and fix javascript check for
 false positive

---
 .../com/safeqr/app/qrcode/entity/URLEntity.java   |  3 +++
 .../qrcode/service/URLVerificationService.java    | 15 ++++++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
index df7192d..9c64dcb 100644
--- a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
+++ b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
@@ -76,6 +76,9 @@ public class URLEntity {
     @Column(name = "tracking_descriptions", columnDefinition = "text[]")
     private List<String> trackingDescriptions = new ArrayList<>();
 
+    @Column(name = "url_encoding")
+    private String urlEncoding = "";
+
     @Column(name = "dns_error")
     private String dnsError = "";
 
diff --git a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
index 5650a9f..9ad3dab 100644
--- a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
+++ b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
@@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLHandshakeException;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
@@ -93,6 +94,9 @@ public class URLVerificationService {
 
             // Check for tracking parameters
             urlObj.setTrackingDescriptions(getTrackingDescriptions(url.getQuery()));
+
+            // Check for URL encoding
+            urlObj.setUrlEncoding(checkURLEncoding(url.getPath()));
         } catch (Exception e) {
             logger.error("Error in breaking down URL: {}", e.getMessage());
         }
@@ -171,7 +175,7 @@ public class URLVerificationService {
         List<Pattern> maliciousPatterns = Arrays.asList(
                 Pattern.compile("javascript:", Pattern.CASE_INSENSITIVE),
                 Pattern.compile("<\\s*script", Pattern.CASE_INSENSITIVE),
-                Pattern.compile("on\\w*\\s*=", Pattern.CASE_INSENSITIVE)
+                Pattern.compile("on(click|mouseover|load|error|unload|submit|reset|focus|blur|change|select|keydown|keyup|keypress|mousedown|mousemove|mouseup|mouseenter|mouseleave|contextmenu|dblclick)\\s*=", Pattern.CASE_INSENSITIVE)
         );
 
         // Check for any malicious pattern in the URL
@@ -191,6 +195,15 @@ public class URLVerificationService {
         return matcher.find() ? "Yes" : "";
     }
 
+    // Function to check text encoding in a URL
+    private static String checkURLEncoding(String pathTextPart) throws UnsupportedEncodingException {
+        // Decode the text
+        String decodedText = URLDecoder.decode(pathTextPart, StandardCharsets.UTF_8.name());
+
+        // Check if the decoded text matches the original text
+        return decodedText.equals(pathTextPart) ? "" : "Yes";
+    }
+
     // Function to detect if the URL has an IP address
     private static String hasIPAddress(String url) {
         Pattern pattern = Pattern.compile(IP_PATTERN, Pattern.CASE_INSENSITIVE);

From 529d27d07c72b7cac68c97e89716a8643fd70f61 Mon Sep 17 00:00:00 2001
From: ltiongku <alex.b.lim@accenture.com>
Date: Tue, 6 Aug 2024 20:21:47 +0800
Subject: [PATCH 5/5] hide empty values in dto and edited check endcoded url
 for false positive

---
 .../safeqr/app/qrcode/entity/URLEntity.java   | 34 +++++++++++++++++--
 .../service/URLVerificationService.java       | 27 ++++++++++-----
 2 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
index 9c64dcb..42b11ec 100644
--- a/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
+++ b/src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java
@@ -1,6 +1,7 @@
 package com.safeqr.app.qrcode.entity;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.hypersistence.utils.hibernate.type.array.ListArrayType;
 import jakarta.persistence.*;
@@ -35,53 +36,82 @@ public class URLEntity {
 
     private String domain;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private String subdomain;
 
     private String topLevelDomain;
 
     private String path;
 
-    @JsonProperty
     private String query;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private String fragment;
 
     private int redirect = 0;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "hsts_header", columnDefinition = "text[]")
     private List<String> hstsHeader = new ArrayList<>();
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "ssl_stripping", columnDefinition = "boolean[]")
     private List<Boolean> sslStripping = new ArrayList<>();
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "redirect_chain", columnDefinition = "text[]")
     private List<String> redirectChain = new ArrayList<>();
 
     @Column(name = "hostname_embedding")
-    private int hostnameEmbedding = 0;
+    private Integer hostnameEmbedding = 0;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Column(name = "javascript_check")
     private String javascriptCheck = "";
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Column(name = "shortening_service")
     private String shorteningService = "";
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Column(name = "has_ip_address")
     private String hasIpAddress = "";
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "tracking_descriptions", columnDefinition = "text[]")
     private List<String> trackingDescriptions = new ArrayList<>();
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Column(name = "url_encoding")
     private String urlEncoding = "";
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Column(name = "dns_error")
     private String dnsError = "";
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Column(name="ssl_error")
     private String sslError = "";
+
+    // Custom getter for hostnameEmbedding
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public Integer getHostnameEmbedding() {
+        return hostnameEmbedding == 0 ? null : hostnameEmbedding;
+    }
+    // Custom getter for path
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public String getPath() {
+        return path.isEmpty() ? null : path;
+    }
+
+    // Custom getter for query
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @JsonProperty
+    public String getQuery() {
+        return query.equals("{}") ? null : query;
+    }
 }
diff --git a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
index 9ad3dab..d355e23 100644
--- a/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
+++ b/src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
@@ -71,7 +71,8 @@ public class URLVerificationService {
     public URLEntity breakdownURL(String urlString) {
         URLEntity urlObj = new URLEntity();
         try {
-            URL url = new URI(encodeUrl(urlString)).toURL();
+            //URL url = new URI(encodeUrl(urlString)).toURL();
+            URL url = new URI(urlString).toURL();
             String host = url.getHost();
 
             // Check for deceptive URL
@@ -88,15 +89,22 @@ public class URLVerificationService {
 
             populateHostDetails(host, urlObj);
 
-            urlObj.setPath(Optional.ofNullable(url.getPath()).filter(p -> !p.isEmpty()).orElse("/"));
-            urlObj.setQuery(parseQueryParams(url.getQuery()));
+            urlObj.setPath(Optional.ofNullable(url.getPath()).filter(p -> !p.isEmpty()).orElse(""));
+
+            String query = parseQueryParams(url.getQuery());
+            urlObj.setQuery(query);
             urlObj.setFragment(Optional.ofNullable(url.getRef()).orElse(""));
 
             // Check for tracking parameters
             urlObj.setTrackingDescriptions(getTrackingDescriptions(url.getQuery()));
 
-            // Check for URL encoding
-            urlObj.setUrlEncoding(checkURLEncoding(url.getPath()));
+            // Check for URL encoding in path and query
+            String pathEncoding = checkURLEncoding(url.getPath());
+            String queryEncoding = query != null ? checkURLEncoding(query) : "";
+
+            // Combine encoding results
+            urlObj.setUrlEncoding(pathEncoding.equals("Yes") || queryEncoding.equals("Yes") ? "Yes" : "");
+
         } catch (Exception e) {
             logger.error("Error in breaking down URL: {}", e.getMessage());
         }
@@ -171,6 +179,9 @@ public class URLVerificationService {
     }
 
     private String checkForJavascriptCode(String url) {
+        // Decode the URL
+        String decodedUrl = URLDecoder.decode(url, StandardCharsets.UTF_8);
+
         // Patterns to detect 'javascript:', '<script>', and 'on*=' attributes
         List<Pattern> maliciousPatterns = Arrays.asList(
                 Pattern.compile("javascript:", Pattern.CASE_INSENSITIVE),
@@ -180,7 +191,7 @@ public class URLVerificationService {
 
         // Check for any malicious pattern in the URL
         for (Pattern pattern : maliciousPatterns) {
-            Matcher matcher = pattern.matcher(url);
+            Matcher matcher = pattern.matcher(decodedUrl);
             if (matcher.find()) {
                 return "Javascript found in URL.";
             }
@@ -196,9 +207,9 @@ public class URLVerificationService {
     }
 
     // Function to check text encoding in a URL
-    private static String checkURLEncoding(String pathTextPart) throws UnsupportedEncodingException {
+    private static String checkURLEncoding(String pathTextPart) {
         // Decode the text
-        String decodedText = URLDecoder.decode(pathTextPart, StandardCharsets.UTF_8.name());
+        String decodedText = URLDecoder.decode(pathTextPart, StandardCharsets.UTF_8);
 
         // Check if the decoded text matches the original text
         return decodedText.equals(pathTextPart) ? "" : "Yes";