define custom setter for URLFeatures

pom.xml

@@ -109,7 +109,38 @@
 			<artifactId>jsoup</artifactId>
 			<version>1.18.1</version>
 		</dependency>
-
+		<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<version>2.17.2</version>
+		</dependency>
+		<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations -->
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-annotations</artifactId>
+			<version>2.17.2</version>
+		</dependency>
+		<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-core_2.13</artifactId>
+			<version>3.4.3</version>
+		</dependency>
+		<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-sql_2.13</artifactId>
+			<version>3.4.3</version>
+			<scope>provided</scope>
+		</dependency>
+		<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-mllib -->
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-mllib_2.13</artifactId>
+			<version>3.4.3</version>
+			<scope>provided</scope>
+		</dependency>
 	</dependencies>
 
 	<build>

src/main/java/com/safeqr/app/constants/APIConstants.java

@@ -23,5 +23,9 @@ public class APIConstants {
     public static final String API_URL_USER_SET_BOOKMARK = "/user/setBookmark";
     public static final String API_URL_USER_DELETE_BOOKMARK = "/user/deleteBookmark";
     public static final String API_URL_USER_DELETE_ALL_BOOKMARK = "/user/deleteAllBookmark";
+    public static final String API_URL_GMAIL_GET_EMAILS = "/gmail/getEmails";
+    public static final String API_URL_GMAIL_GET_SCANNED_EMAILS = "/gmail/getScannedEmails";
+
+
 
 }

src/main/java/com/safeqr/app/constants/CommonConstants.java

@@ -17,4 +17,9 @@ public class CommonConstants {
     public static final String INFO_NO_HSTS_HEADER = "No HSTS Header detected";
     public static final String INFO_HSTS_HEADER_PREFIX = "HSTS Header: ";
     public static final String INFO_HSTS_NOT_APPLICABLE = "N/A";
+
+    public static final String CLASSIFY_SAFE = "SAFE";
+    public static final String CLASSIFY_WARNING = "WARNING";
+    public static final String CLASSIFY_UNSAFE = "UNSAFE";
+    public static final String CLASSIFY_UNKNOWN = "UNKNOWN";
 }

src/main/java/com/safeqr/app/gmail/controller/GmailController.java

@@ -1,6 +1,7 @@
 package com.safeqr.app.gmail.controller;
 
 import com.google.api.services.gmail.model.*;
+import com.safeqr.app.gmail.dto.ScannedGmailResponseDto;
 import org.apache.commons.codec.binary.Base64;
 import org.json.JSONObject;
 import com.google.api.client.auth.oauth2.AuthorizationCodeRequestUrl;
@@ -26,8 +27,10 @@ import org.springframework.web.bind.annotation.*;
 import org.springframework.web.servlet.view.RedirectView;
 import static com.safeqr.app.constants.APIConstants.*;
 import java.io.IOException;
+import java.lang.Thread;
 import java.util.*;
-
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 
 
 @RestController
@@ -99,14 +102,32 @@ public class GmailController {
         return new ResponseEntity<>(json.toString(), HttpStatus.OK);
     }
 
-    @GetMapping(value = "/gmail/getEmails", produces = MediaType.APPLICATION_JSON_VALUE)
+    @GetMapping(value = API_URL_GMAIL_GET_SCANNED_EMAILS, produces = MediaType.APPLICATION_JSON_VALUE)
-    public ResponseEntity<?> getUserEmails(@RequestHeader(name = "accessToken") String accessToken) throws IOException, InterruptedException {
+    public ResponseEntity<ScannedGmailResponseDto> getUserScannedEmails(@RequestHeader(name = "X-USER-ID") String userId) {
-        logger.info("Invoking GET Scan User Emails endpoints");
+        logger.info("User Id Invoking GET User scanned Emails endpoint: {}", userId);
+        return ResponseEntity.ok(gmailService.fetchScannedGmail(userId));
+    }
+    @GetMapping(value = API_URL_GMAIL_GET_EMAILS, produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<?> getUserEmails(@RequestHeader(name = "accessToken") String accessToken,
+                                           @RequestHeader(name = "refreshToken") String refreshToken,
+                                           @RequestHeader(name = "X-USER-ID") String userId
+                                           )  {
+        logger.info("User Id Invoking GET Scan User Emails endpoints: {}", userId);
         if (accessToken == null || accessToken.isEmpty()) {
             return new ResponseEntity<>("Access token is missing", HttpStatus.BAD_REQUEST);
         }
+        logger.info("accessToken -> {}", accessToken);
+        logger.info("refreshToken -> {}", refreshToken);
+        logger.info("userId -> {}", userId);
 
-        return new ResponseEntity<>(gmailService.getEmail(accessToken).toString(), HttpStatus.OK);
+        CompletableFuture.runAsync(() -> {
+            gmailService.getEmailAsync(userId, accessToken, refreshToken);
+        }).exceptionally(throwable -> {
+            logger.error("Unexpected error occurred while processing emails", throwable);
+            return null;
+        });
+
+        return new ResponseEntity<>("Scan Gmail Request is being processed", HttpStatus.ACCEPTED);
     }
 }
 

src/main/java/com/safeqr/app/gmail/dto/ScannedGmailResponseDto.java

@@ -0,0 +1,15 @@
+package com.safeqr.app.gmail.dto;
+
+import com.safeqr.app.gmail.model.EmailMessage;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Builder
+@Data
+@AllArgsConstructor
+public class ScannedGmailResponseDto {
+    List<EmailMessage> messages;
+}

src/main/java/com/safeqr/app/gmail/entity/GmailCidEntity.java

@@ -0,0 +1,40 @@
+package com.safeqr.app.gmail.entity;
+
+import com.safeqr.app.qrcode.entity.QRCodeEntity;
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.annotations.UuidGenerator;
+
+import java.util.UUID;
+
+@Entity
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@Table(name = "gmail_cid", schema = "safeqr")
+public class GmailCidEntity {
+    @Id
+    @GeneratedValue(generator = "UUID")
+    @UuidGenerator
+    @Column(updatable = false, nullable = false)
+    private UUID id;
+
+    @Column(name = "gmail_id")
+    private UUID gmailId;
+
+    @Column(name = "cid")
+    private String cid;
+
+    @Column(name = "attachment_id")
+    private String attachmentId;
+
+    @Column(name = "decoded_content")
+    private String decodedContent;
+
+    @Column(name = "qr_code_id")
+    private UUID qrCodeId;
+}

src/main/java/com/safeqr/app/gmail/entity/GmailEmailEntity.java

@@ -0,0 +1,52 @@
+package com.safeqr.app.gmail.entity;
+
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.annotations.UuidGenerator;
+
+import java.time.OffsetDateTime;
+import java.util.UUID;
+
+@Entity
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@Table(name = "gmail_emails", schema = "safeqr")
+public class GmailEmailEntity {
+    @Id
+    @GeneratedValue(generator = "UUID")
+    @UuidGenerator
+    @Column(updatable = false, nullable = false)
+    private UUID id;
+
+    @Column(name = "user_id")
+    private String userId;
+
+    @Column(name = "message_id")
+    private String messageId;
+
+    @Column(name = "thread_id")
+    private String threadId;
+
+    @Column(name = "history_id")
+    private Long historyId;
+
+    @Column(name= "subject")
+    private String subject;
+
+    @Column(name = "date_received")
+    private OffsetDateTime dateReceived;
+
+    @Column(name = "date_created")
+    private OffsetDateTime dateCreated;
+
+    @PrePersist
+    public void prePersist() {
+        dateCreated = OffsetDateTime.now();
+    }
+
+}

src/main/java/com/safeqr/app/gmail/entity/GmailUrlEntity.java

@@ -0,0 +1,38 @@
+package com.safeqr.app.gmail.entity;
+
+import com.safeqr.app.qrcode.entity.QRCodeEntity;
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.annotations.UuidGenerator;
+
+import java.util.UUID;
+
+@Entity
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@Table(name = "gmail_urls", schema = "safeqr")
+public class GmailUrlEntity {
+    @Id
+    @GeneratedValue(generator = "UUID")
+    @UuidGenerator
+    @Column(updatable = false, nullable = false)
+    private UUID id;
+
+    @Column(name = "gmail_id")
+    private UUID gmailId;
+
+    @Column(name = "image_url")
+    private String imageUrl;
+
+    @Column(name = "decoded_content")
+    private String decodedContent;
+
+    @Column(name = "qr_code_id")
+    private UUID qrCodeId;
+
+}

src/main/java/com/safeqr/app/gmail/model/EmailMessage.java

@@ -0,0 +1,51 @@
+package com.safeqr.app.gmail.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.safeqr.app.qrcode.model.QRCodeModel;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+@Data
+public class EmailMessage {
+    private String messageId;
+    private String threadId;
+    private String subject;
+    private String historyId;
+    private String date;
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    List<QRCodeByContentId> qrCodeByContentId;
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    List<QRCodeByURL> qrCodeByURL;
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    List<QRCodeModel<?>> decodedContentsDetails;
+
+    public EmailMessage(String messageId, String threadId ,String subject, String historyId, String date) {
+        this.messageId = messageId;
+        this.threadId = threadId;
+        this.subject = subject;
+        this.historyId = historyId;
+        this.date = date;
+        this.qrCodeByContentId = new ArrayList<>();
+        this.qrCodeByURL = new ArrayList<>();
+        this.decodedContentsDetails = new ArrayList<>();
+    }
+    public void addQRCodeByContentId(QRCodeByContentId qrCode) {
+        this.qrCodeByContentId.add(qrCode);
+    }
+
+    public void addQRCodeByURL(QRCodeByURL qrCode) {
+        this.qrCodeByURL.add(qrCode);
+    }
+
+    public void addQRCodeModel(QRCodeModel<?> qrCode) {
+        this.decodedContentsDetails.add(qrCode);
+    }
+
+    public boolean hasQRCodes() {
+        return !qrCodeByContentId.isEmpty() || !qrCodeByURL.isEmpty();
+    }
+}

src/main/java/com/safeqr/app/gmail/model/QRCodeByContentId.java

@@ -0,0 +1,22 @@
+package com.safeqr.app.gmail.model;
+
+import com.safeqr.app.qrcode.model.QRCodeModel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+@AllArgsConstructor
+public class QRCodeByContentId {
+    private String cid;
+    private String attachmentId;
+    private List<String> decodedContent;
+    private int totalQRCodeFound;
+
+    public List<String> getDecodedContent() {
+        return decodedContent;
+    }
+}

src/main/java/com/safeqr/app/gmail/model/QRCodeByURL.java

@@ -0,0 +1,17 @@
+package com.safeqr.app.gmail.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+@AllArgsConstructor
+public class QRCodeByURL {
+    private String url;
+    private List<String> decodedContent;
+    private int totalQRCodeFound;
+
+}

src/main/java/com/safeqr/app/gmail/repository/GmailCidRespository.java

@@ -0,0 +1,11 @@
+package com.safeqr.app.gmail.repository;
+
+import com.safeqr.app.gmail.entity.GmailCidEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import java.util.List;
+import java.util.UUID;
+
+public interface GmailCidRespository extends JpaRepository<GmailCidEntity, UUID> {
+    List<GmailCidEntity> findByGmailId(UUID gmailId);
+}

src/main/java/com/safeqr/app/gmail/repository/GmailEmailRespository.java

@@ -0,0 +1,12 @@
+package com.safeqr.app.gmail.repository;
+
+
+import com.safeqr.app.gmail.entity.GmailEmailEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import java.util.List;
+import java.util.UUID;
+
+public interface GmailEmailRespository extends JpaRepository<GmailEmailEntity, UUID> {
+    List<GmailEmailEntity> findByUserId(String userId);
+}

src/main/java/com/safeqr/app/gmail/repository/GmailUrlsRespository.java

@@ -0,0 +1,11 @@
+package com.safeqr.app.gmail.repository;
+
+import com.safeqr.app.gmail.entity.GmailUrlEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import java.util.List;
+import java.util.UUID;
+
+public interface GmailUrlsRespository extends JpaRepository<GmailUrlEntity, UUID> {
+    List<GmailUrlEntity> findByGmailId(UUID gmailId);
+}

src/main/java/com/safeqr/app/gmail/service/GmailService.java

@@ -1,119 +1,412 @@
 package com.safeqr.app.gmail.service;
 
 import com.google.api.client.auth.oauth2.BearerToken;
+import com.google.api.client.auth.oauth2.ClientParametersAuthentication;
+import com.google.api.client.auth.oauth2.Credential;
+
+import com.google.api.client.auth.oauth2.TokenResponse;
+import com.google.api.client.googleapis.auth.oauth2.GoogleRefreshTokenRequest;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.services.gmail.Gmail;
 import com.google.api.services.gmail.model.*;
 import com.google.zxing.*;
 import com.google.zxing.client.j2se.BufferedImageLuminanceSource;
 import com.google.zxing.common.HybridBinarizer;
 import com.google.zxing.multi.qrcode.QRCodeMultiReader;
+import com.google.zxing.qrcode.encoder.QRCode;
+import com.safeqr.app.gmail.dto.ScannedGmailResponseDto;
+import com.safeqr.app.gmail.entity.GmailCidEntity;
+import com.safeqr.app.gmail.entity.GmailEmailEntity;
+import com.safeqr.app.gmail.entity.GmailUrlEntity;
+import com.safeqr.app.gmail.model.EmailMessage;
+import com.safeqr.app.gmail.model.QRCodeByContentId;
+import com.safeqr.app.gmail.model.QRCodeByURL;
+import com.safeqr.app.gmail.repository.GmailCidRespository;
+import com.safeqr.app.gmail.repository.GmailEmailRespository;
+import com.safeqr.app.gmail.repository.GmailUrlsRespository;
+import com.safeqr.app.qrcode.model.QRCodeModel;
+import com.safeqr.app.qrcode.service.QRCodeTypeService;
+import com.safeqr.app.user.service.UserService;
+import com.safeqr.app.utils.DateParsingUtils;
 import org.apache.commons.codec.binary.Base64;
-import org.json.JSONArray;
-import org.json.JSONObject;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.dao.DataIntegrityViolationException;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.gson.GsonFactory;
-import com.google.api.services.gmail.Gmail;
 
 import javax.imageio.ImageIO;
 import java.awt.image.BufferedImage;
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.lang.Thread;
+import java.net.ConnectException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.net.http.HttpTimeoutException;
+import java.time.OffsetDateTime;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static com.google.api.client.googleapis.auth.oauth2.GoogleOAuthConstants.TOKEN_SERVER_URL;
 import static com.safeqr.app.constants.APIConstants.APPLICATION_NAME;
 
 @Service
 public class GmailService {
+    @Value("${gmail.client.clientId}")
+    private String clientId;
+
+    @Value("${gmail.client.clientSecret}")
+    private String clientSecret;
+
+    private final GmailEmailRespository gmailEmailRespository;
+    private final GmailCidRespository gmailCidRespository;
+    private final GmailUrlsRespository gmailUrlsRespository;
+    private final QRCodeTypeService qrCodeTypeService;
+    private final UserService userService;
     private static final Logger logger = LoggerFactory.getLogger(GmailService.class);
     private static final HttpTransport httpTransport = new NetHttpTransport();
     private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
+    private static final long MAX_RESULTS = 100L;
+
+    public GmailService(GmailEmailRespository gmailEmailRespository,
+                        GmailCidRespository gmailCidRespository,
+                        GmailUrlsRespository gmailUrlsRespository,
+                        QRCodeTypeService qrCodeTypeService,
+                        UserService userService) {
+        this.gmailEmailRespository = gmailEmailRespository;
+        this.gmailCidRespository = gmailCidRespository;
+        this.gmailUrlsRespository = gmailUrlsRespository;
+        this.qrCodeTypeService = qrCodeTypeService;
+        this.userService = userService;
+    }
+
+    private Gmail getGmailService(String accessToken, String refreshToken) throws IOException {
+        Credential userCredentials = new Credential.Builder(BearerToken.authorizationHeaderAccessMethod())
+                .setTransport(httpTransport)
+                .setJsonFactory(JSON_FACTORY)
+                .setTokenServerUrl(new GenericUrl(TOKEN_SERVER_URL))
+                .setClientAuthentication(new ClientParametersAuthentication(clientId, clientSecret))
+                .build()
+                .setAccessToken(accessToken)
+                .setRefreshToken(refreshToken);
 
-    private Gmail getGmailService(String accessToken) {
-        Credential userCredentials = new Credential(BearerToken.authorizationHeaderAccessMethod()).setAccessToken(accessToken);
         return new Gmail.Builder(httpTransport, JSON_FACTORY, userCredentials)
                 .setApplicationName(APPLICATION_NAME)
                 .build();
     }
+    // Renew the access token if it has expired using the refresh token.
+    private String refreshAccessToken(String refreshToken) throws IOException {
+        TokenResponse response = new GoogleRefreshTokenRequest(
+                httpTransport, JSON_FACTORY, refreshToken, clientId, clientSecret)
+                .execute();
+        return response.getAccessToken();
+    }
 
-    public JSONObject getEmail(String accessToken) throws IOException, InterruptedException {
+    private Gmail refreshAndGetGmailService(String accessToken, String refreshToken) throws IOException {
-        JSONObject json = new JSONObject();
+        try {
-        JSONArray emailArray = new JSONArray();
+            return getGmailService(accessToken, refreshToken);
+        } catch (GoogleJsonResponseException e) {
+            if (e.getStatusCode() == 401) {
+                logger.info("Access token expired. Refreshing token...");
+                String newAccessToken = refreshAccessToken(refreshToken);
+                return getGmailService(newAccessToken, refreshToken);
+            }
+            throw e;
+        }
+    }
 
-        // Build the Gmail service
+    // Async method to scan all emails in the user's inbox to prevent timeout.
-        Gmail service = getGmailService(accessToken);
+    @Async
-        logger.info("service-> {}", service);
+    public void getEmailAsync(String userId, String accessToken, String refreshToken) {
+        try {
+            ScannedGmailResponseDto result = getEmail(userId, accessToken, refreshToken);
+            CompletableFuture.completedFuture(result);
+        } catch (IOException e) {
+            logger.error("Error processing Gmail", e);
+        }
+    }
+    // Scan all emails in the user's inbox.
+    public ScannedGmailResponseDto getEmail(String userId, String accessToken, String refreshToken) throws IOException {
+        Gmail service = refreshAndGetGmailService(accessToken, refreshToken);
+        logger.info("Gmail service initialized: {}", service);
 
-        // Get the list of messages
+        List<EmailMessage> emailMessagesList = new ArrayList<>();
-        ListMessagesResponse listResponse = service.users().messages().list("me").execute();
+        String meUserId = "me";
-        List<Message> messages = listResponse.getMessages();
+        String nextPageToken = null;
+        // Fetching email messages with page token and setting max results, Default value is 100.
+        do {
 
-        for (Message message : messages) {
+//            ListHistoryResponse historyResponse = service.users().history().list(meUserId)
-            message = service.users().messages().get("me", message.getId()).setFormat("full").execute();
+//                    .setStartHistoryId(BigInteger.valueOf(689335))
-            List<MessagePart> parts = message.getPayload().getParts();
+//                    .execute();
-            Set<String> attachmentIds = new HashSet<>();
+//
-            Set<String> imageUrls = new HashSet<>();
+//            List<History> historyList = historyResponse.getHistory();
-            processPartsRecursively(parts, attachmentIds, imageUrls);
+//
+//            for (History history : historyList) {
+//                logger.info("History Id: {}, Message Id: {}, Message Snippet: {}", history.getId(), history.getMessages().get(0).getId(), history.getMessages().get(0).getHistoryId());
+//            }
+            ListMessagesResponse listResponse = fetchMessages(service, meUserId, nextPageToken);
+            List<Message> messages = listResponse.getMessages();
+            nextPageToken = listResponse.getNextPageToken();
 
-            // Extract and log the email subject
+            // Iterate all the messages and save to gmail db only if it has a valid QR code.
-            String subject = getSubject(message);
+            for (Message message : messages) {
-            logger.info("Email Subject-> {}", subject);
+                EmailMessage emailMessage = processMessage(service, meUserId, message);
-
+                if (emailMessage != null) {
-            if (attachmentIds.isEmpty() && imageUrls.isEmpty())
+                    emailMessagesList.add(emailMessage);
-                continue;
+                    // Save email message to database.
-
+                    saveEmailMessageAndScanQRCode(userId, emailMessage);
-            String messageId = message.getId();
-            logger.info("messageId-> {}", messageId);
-            String historyId = String.valueOf(message.getHistoryId());
-            logger.info("historyId-> {}", historyId);
-
-            for (String attachmentId : attachmentIds) {
-                Optional<String> attachment = findAttachmentIdByCid(parts, attachmentId);
-                logger.info("attachment-> {}", attachment);
-                if (attachment.isPresent()) {
-                    List<String> qrCodeValue = processAttachment(service, messageId, attachment.get());
-                    emailArray.put(qrCodeValue);
                 }
             }
-            for (String imageUrl : imageUrls) {
+        } while (nextPageToken != null);
-                List<String> qrCodeValue = scanQRCodeFromUrl(imageUrl);
+
-                if (qrCodeValue != null) {
+        // Update user's history id.
-                    emailArray.put(qrCodeValue);
+        // TODO: Update user's history id.
+
+        return new ScannedGmailResponseDto(emailMessagesList);
+    }
+    // Save email message to database and scan QR code.
+    private void saveEmailMessageAndScanQRCode(String userId, EmailMessage emailMessage) {
+        GmailEmailEntity gmailEmailEntity = saveEmailMessage(userId, emailMessage);
+
+        if (gmailEmailEntity != null) {
+            // Save QR codes by content ID
+            saveQRCodeByContentId(gmailEmailEntity, emailMessage.getQrCodeByContentId());
+
+            // Save QR codes by URL
+            saveQRCodeByURL(gmailEmailEntity, emailMessage.getQrCodeByURL());
+        } else {
+            logger.warn("Skipping QR code processing due to failure in saving email message.");
+        }
+    }
+    // Save to gmail_email table
+    private GmailEmailEntity saveEmailMessage(String userId, EmailMessage emailMessage) {
+        logger.info("userId: {}", userId);
+        OffsetDateTime dateReceived = DateParsingUtils.parseDate(emailMessage.getDate());
+        try {
+            GmailEmailEntity gmailEmailEntity = GmailEmailEntity.builder()
+                    .userId(userId)
+                    .messageId(emailMessage.getMessageId())
+                    .threadId(emailMessage.getThreadId())
+                    .historyId(Long.valueOf(emailMessage.getHistoryId()))
+                    .subject(emailMessage.getSubject())
+                    .dateReceived(dateReceived)
+                    .build();
+            return gmailEmailRespository.save(gmailEmailEntity);
+        } catch (DataIntegrityViolationException e) {
+            if (e.getCause() instanceof org.hibernate.exception.ConstraintViolationException) {
+                logger.warn("Duplicate entry for userId: {}, messageId: {}", userId, emailMessage.getMessageId());
+            } else {
+                logger.error("Error saving to gmail_email table: {}", e.getMessage(), e);
+            }
+        } catch (Exception e) {
+            logger.error("Error saving gmail_email table: {}", e.getMessage(), e);
+        }
+        return null;
+    }
+
+    // Iterate through decoded contents found in attachment as content id and save to gmail_cid table
+    private void saveQRCodeByContentId(GmailEmailEntity gmailEmailEntity, List<QRCodeByContentId> qrCodeByContentIdList) {
+        qrCodeByContentIdList.forEach(cid -> {
+            cid.getDecodedContent().forEach(decodedContent -> {
+                try {
+                    QRCodeModel<?> qrCodeModel = qrCodeTypeService.scanGmailDecodedContents(gmailEmailEntity.getUserId(), decodedContent);
+                    GmailCidEntity gmailCidEntity = GmailCidEntity.builder()
+                            .gmailId(gmailEmailEntity.getId())
+                            .qrCodeId(qrCodeModel.getData().getId())
+                            .cid(cid.getCid())
+                            .attachmentId(cid.getAttachmentId())
+                            .decodedContent(decodedContent)
+                            .build();
+                    gmailCidRespository.save(gmailCidEntity);
+                } catch (Exception e) {
+                    logger.error("Error saving QR code by content ID to gmail_cid table: {}", e.getMessage(), e);
+                }
+            });
+        });
+    }
+    // Iterate through decoded content found in url and save to gmail_url table
+    private void saveQRCodeByURL(GmailEmailEntity gmailEmailEntity, List<QRCodeByURL> qrCodeByURLList) {
+        qrCodeByURLList.forEach(imageUrl -> {
+            imageUrl.getDecodedContent().forEach(decodedContent -> {
+                try {
+                    QRCodeModel<?> qrCodeModel = qrCodeTypeService.scanGmailDecodedContents(gmailEmailEntity.getUserId(), decodedContent);
+                    GmailUrlEntity gmailUrlEntity = GmailUrlEntity.builder()
+                            .gmailId(gmailEmailEntity.getId())
+                            .qrCodeId(qrCodeModel.getData().getId())
+                            .imageUrl(imageUrl.getUrl())
+                            .decodedContent(decodedContent)
+                            .build();
+                    gmailUrlsRespository.save(gmailUrlEntity);
+                } catch (Exception e) {
+                    logger.error("Error saving QR code by URL to gmail_urls table: {}", e.getMessage(), e);
+                }
+            });
+        });
+    }
+
+    // Fetching Scanned Gmail from database
+    public ScannedGmailResponseDto fetchScannedGmail(String userId){
+        // Fetching all emails from gmail_email table
+        List<GmailEmailEntity> userEmailsList = gmailEmailRespository.findByUserId(userId);
+        List<EmailMessage> emailMessageList = new ArrayList<>();
+
+        if (userEmailsList != null && !userEmailsList.isEmpty()) {
+            userEmailsList.forEach(email -> {
+                EmailMessage emailMessage = new EmailMessage(
+                        email.getMessageId(),
+                        email.getThreadId(),
+                        email.getSubject(),
+                        email.getHistoryId().toString(),
+                        email.getDateReceived().toString()
+                );
+                // Fetching all CIDs from gmail_cid table
+                List<GmailCidEntity> cidList = gmailCidRespository.findByGmailId(email.getId());
+                Map<String, QRCodeByContentId> qrCodeByContentIdMap = new HashMap<>();
+
+                for (GmailCidEntity cid : cidList) {
+                    String key = cid.getCid() + "-" + cid.getAttachmentId();
+                    QRCodeByContentId qrCodeByContentId = qrCodeByContentIdMap.get(key);
+
+                    if (qrCodeByContentId == null) {
+                        qrCodeByContentId = QRCodeByContentId.builder()
+                                .cid(cid.getCid())
+                                .attachmentId(cid.getAttachmentId())
+                                .decodedContent(new ArrayList<>())
+                                .totalQRCodeFound(0)
+                                .build();
+                        qrCodeByContentIdMap.put(key, qrCodeByContentId);
+                    }
+
+                    // Append decoded content to the existing list
+                    qrCodeByContentId.getDecodedContent().add(cid.getDecodedContent());
+
+                    // Fetch scanned QR code from database and add to message object
+                    emailMessage.addQRCodeModel(qrCodeTypeService.getScannedQRCodeDetailsInModel(cid.getQrCodeId()));
+
+                    qrCodeByContentId.setTotalQRCodeFound(qrCodeByContentId.getTotalQRCodeFound() + 1);
+                }
+
+                emailMessage.setQrCodeByContentId(new ArrayList<>(qrCodeByContentIdMap.values()));
+
+                // Fetching all URLs from gmail_urls table
+                List<GmailUrlEntity> urlList = gmailUrlsRespository.findByGmailId(email.getId());
+
+                Map<String, QRCodeByURL> qrCodeByURLMap = new HashMap<>();
+
+                for (GmailUrlEntity url : urlList) {
+                    String key = url.getImageUrl();
+                    QRCodeByURL qrCodeByURL = qrCodeByURLMap.get(key);
+
+                    if (qrCodeByURL == null) {
+                        qrCodeByURL = QRCodeByURL.builder()
+                                .url(url.getImageUrl())
+                                .decodedContent(new ArrayList<>())
+                                .totalQRCodeFound(0)
+                                .build();
+                        qrCodeByURLMap.put(key, qrCodeByURL);
+                    }
+
+                    // Append decoded content to the existing list
+                    qrCodeByURL.getDecodedContent().add(url.getDecodedContent());
+
+                    // Fetch scanned QR code from database and add to message object
+                    emailMessage.addQRCodeModel(qrCodeTypeService.getScannedQRCodeDetailsInModel(url.getQrCodeId()));
+                    qrCodeByURL.setTotalQRCodeFound(qrCodeByURL.getTotalQRCodeFound() + 1);
+                }
+
+                emailMessage.setQrCodeByURL(new ArrayList<>(qrCodeByURLMap.values()));
+
+                emailMessageList.add(emailMessage);
+
+            });
+        }
+        return ScannedGmailResponseDto.builder().messages(emailMessageList).build();
+    }
+    // Fetching email messages with page token and setting max results
+    private ListMessagesResponse fetchMessages(Gmail service, String userId, String pageToken) throws IOException {
+        return service.users().messages().list(userId)
+                .setPageToken(pageToken)
+                .setMaxResults(MAX_RESULTS)
+                .execute();
+    }
+    // Processing email message and returning EmailMessage object if it has a valid QR code.
+    private EmailMessage processMessage(Gmail service, String userId, Message message) throws IOException {
+        message = service.users().messages().get(userId, message.getId()).setFormat("full").execute();
+        List<MessagePart> parts = message.getPayload().getParts();
+        Set<String> attachmentIds = new HashSet<>();
+        Set<String> imageUrls = new HashSet<>();
+        processPartsRecursively(parts, attachmentIds, imageUrls);
+
+        if (attachmentIds.isEmpty() && imageUrls.isEmpty()) {
+            return null;
+        }
+
+        String subject = getHeader(message, "Subject");
+        String emailDate = getHeader(message, "Date");
+        logger.info("Email Subject: {}", subject);
+        logger.info("Message ID: {}", message.getId());
+        logger.info("History ID: {}", message.getHistoryId());
+
+        EmailMessage emailMessage = new EmailMessage(message.getId(), message.getThreadId(), subject, String.valueOf(message.getHistoryId()), emailDate);
+
+        processAttachments(service, message.getId(), parts, attachmentIds, emailMessage);
+        processImageUrls(imageUrls, emailMessage);
+
+        return emailMessage.hasQRCodes() ? emailMessage : null;
+    }
+    // Process all the attachments.
+    private void processAttachments(Gmail service, String messageId, List<MessagePart> parts, Set<String> attachmentIds, EmailMessage emailMessage) throws IOException {
+        for (String attachmentId : attachmentIds) {
+            Optional<String> attachment = findAttachmentIdByCid(parts, attachmentId);
+            if (attachment.isPresent()) {
+                List<String> qrCodeValue = processAttachment(service, messageId, attachment.get());
+                if (!qrCodeValue.isEmpty()) {
+                    emailMessage.addQRCodeByContentId(new QRCodeByContentId(attachmentId, attachment.get(), qrCodeValue, qrCodeValue.size()));
                 }
             }
         }
-        logger.info("Total Emails-> {}", messages.size());
-        json.put("qr_codes", emailArray);
-        return json;
     }
-
+    // Process all the image URLs.
-    private String getSubject(Message message) {
+    private void processImageUrls(Set<String> imageUrls, EmailMessage emailMessage)  {
+        for (String imageUrl : imageUrls) {
+            List<String> qrCodeValue = scanQRCodeFromUrl(imageUrl);
+            if (!qrCodeValue.isEmpty()) {
+                emailMessage.addQRCodeByURL(new QRCodeByURL(imageUrl, qrCodeValue, qrCodeValue.size()));
+            }
+        }
+    }
+    // Find the header with the given name.
+    private String getHeader(Message message, String name) {
         return message.getPayload().getHeaders().stream()
-                .filter(header -> "Subject".equals(header.getName()))
+                .filter(header -> name.equalsIgnoreCase(header.getName()))
                 .findFirst()
                 .map(MessagePartHeader::getValue)
-                .orElse("No Subject");
+                .orElse("No " + name);
     }
+    // Find the attachment ID in the given message part.
     private Optional<String> findAttachmentIdByCid(List<MessagePart> parts, String cid) {
         return parts.stream()
                 .flatMap(part -> Stream.concat(findAttachmentIdInCurrentPart(part, cid).stream(), Optional.ofNullable(part.getParts())
                 .flatMap(subParts -> findAttachmentIdByCid(subParts, cid)).stream()))
                 .findFirst();
     }
-
+    // Find the attachment ID in the message subpart.
     private Optional<String> findAttachmentIdInCurrentPart(MessagePart part, String cid) {
         return Optional.ofNullable(part.getHeaders())
                 .flatMap(headers -> headers.stream()
@@ -121,7 +414,7 @@ public class GmailService {
                 .findFirst()
                 .map(header -> part.getBody().getAttachmentId()));
     }
-
+    // Check if the header is a Content-ID header with the given CID.
     private boolean isContentIdHeader(MessagePartHeader header, String cid) {
         return "Content-ID".equalsIgnoreCase(header.getName()) && header.getValue().contains(cid);
     }
@@ -140,35 +433,55 @@ public class GmailService {
             }
         }
     }
-    private List<String> scanQRCodeFromUrl(String imageUrl) throws IOException, InterruptedException {
+    private List<String> scanQRCodeFromUrl(String imageUrl) {
         try {
             BufferedImage image = downloadImageFromUrl(imageUrl);
             if (image != null) {
                 return decodeQRCodes(image);
             }
+        } catch (IllegalArgumentException e) {
+            logger.error("Invalid URI scheme for URL: {} -> {}", imageUrl, e.getMessage());
         } catch(URISyntaxException e) {
             logger.error("Error while scanning QR code from URL", e);
         }
-        return null;
+        return Collections.emptyList();
     }
     // Download the image from the given URL
-    private BufferedImage downloadImageFromUrl(String imageUrl) throws IOException, InterruptedException, URISyntaxException {
+    private BufferedImage downloadImageFromUrl(String imageUrl) throws URISyntaxException {
-        HttpClient client = HttpClient.newBuilder()
+        try {
-                .followRedirects(HttpClient.Redirect.ALWAYS)
+            imageUrl = imageUrl.replace(" ", "%20");
-                .build();
+            HttpClient client = HttpClient.newBuilder()
-        logger.info("imageUrl-> {}", imageUrl);
+                    .followRedirects(HttpClient.Redirect.ALWAYS)
-        // Encode the URL
+                    .build();
-        HttpRequest request = HttpRequest.newBuilder()
+            logger.info("Downloading image from URL: {}", imageUrl);
-                .uri(URI.create(imageUrl.replace(" ", "%20")))
+            HttpRequest request = HttpRequest.newBuilder()
-                .GET()
+                    .uri(new URI(imageUrl))
-                .build();
+                    .header("User-Agent", "Mozilla/5.0")
+                    .GET()
+                    .build();
 
-        HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
+            HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
-        if (response.statusCode() == 200) {
+
-            byte[] imageBytes = response.body();
+            if (response.statusCode() == 200) {
-            return ImageIO.read(new ByteArrayInputStream(imageBytes));
+                byte[] imageBytes = response.body();
-        } else {
+                return ImageIO.read(new ByteArrayInputStream(imageBytes));
-            logger.error("Failed to download image. HTTP response code: {}", response.statusCode());
+            } else {
+                logger.warn("Failed to download image. HTTP response code: {}", response.statusCode());
+            }
+        } catch (URISyntaxException e) {
+            logger.error("Invalid URL: {} -> {}", imageUrl, e.getMessage());
+        } catch (HttpTimeoutException e) {
+            logger.error("Request timed out for URL: {} -> {}", imageUrl, e.getMessage());
+        } catch (ConnectException e) {
+            logger.warn("Failed to connect to URL: {} -> {}", imageUrl, e.getMessage());
+        } catch (IOException e) {
+            logger.warn("Error downloading image from URL: {} -> {}", imageUrl, e.getMessage());
+            if (Thread.currentThread().isInterrupted()) {
+                logger.warn("Thread was interrupted during IO operation for URL: {}", imageUrl);
+            }
+        } catch (InterruptedException e) {
+            logger.warn("Thread was interrupted during HTTP request for URL: {} -> {}", imageUrl, e.getMessage());
+            Thread.currentThread().interrupt();
         }
         return null;
     }
@@ -199,16 +512,14 @@ public class GmailService {
                     qrCodeValues.add(result.getText());
                     logger.info("Detected QR code: {}", result.getText());
                 }
-            } else {
-                logger.info("No QR codes found in the image");
             }
         } catch (NotFoundException e) {
-            logger.info("No QR codes found in the image");
+            // No QR codes found
         } catch (Exception e) {
             logger.error("Error decoding QR codes", e);
         }
-
+        if (!qrCodeValues.isEmpty())
-        logger.info("Total QR codes found: {}", qrCodeValues.size());
+            logger.info("Total QR codes found: {}", qrCodeValues.size());
         return qrCodeValues;
     }
 
@@ -238,5 +549,4 @@ public class GmailService {
         String lowerUrl = url.toLowerCase();
         return lowerUrl.endsWith(".jpg") || lowerUrl.endsWith(".jpeg") || lowerUrl.endsWith(".png") || lowerUrl.endsWith(".gif") || lowerUrl.endsWith(".bmp");
     }
-
 }

src/main/java/com/safeqr/app/qrcode/controller/QRCodeTypeController.java

@@ -52,7 +52,7 @@ public class QRCodeTypeController {
     @PostMapping(value = API_URL_QRCODE_SCAN, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<BaseScanResponse> scanQRCode(@RequestBody QRCodePayload payload,
                                                        @RequestHeader(required = false, name = HEADER_USER_ID) String userId) {
-        logger.info("Invoking scan endpoint");
+        logger.info("User Id Invoking scan endpoint: {}", userId);
         return ResponseEntity.ok(qrCodeTypeService.scanQRCode(userId, payload));
     }
 

src/main/java/com/safeqr/app/qrcode/entity/QRCodeEntity.java

@@ -12,6 +12,8 @@ import org.hibernate.annotations.UuidGenerator;
 import java.time.LocalDateTime;
 import java.util.UUID;
 
+import static com.safeqr.app.constants.CommonConstants.CLASSIFY_UNKNOWN;
+
 @Entity
 @Table(name = "qr_code", schema = "safeqr")
 @Data
@@ -36,4 +38,7 @@ public class QRCodeEntity {
 
     @Column(name = "created_at", insertable = false, updatable = false)
     private LocalDateTime createdAt;
+
+    @Column(name = "result_category")
+    private String result = CLASSIFY_UNKNOWN;
 }

src/main/java/com/safeqr/app/qrcode/entity/URLEntity.java

@@ -1,6 +1,7 @@
 package com.safeqr.app.qrcode.entity;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.hypersistence.utils.hibernate.type.array.ListArrayType;
 import jakarta.persistence.*;
@@ -11,6 +12,7 @@ import lombok.Builder;
 import org.hibernate.annotations.Type;
 import org.hibernate.annotations.UuidGenerator;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 
@@ -34,28 +36,86 @@ public class URLEntity {
 
     private String domain;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private String subdomain;
 
     private String topLevelDomain;
 
     private String path;
 
-    @JsonProperty
     private String query;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private String fragment;
 
     private int redirect = 0;
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "hsts_header", columnDefinition = "text[]")
-    private List<String> hstsHeader;
+    private List<String> hstsHeader = new ArrayList<>();
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "ssl_stripping", columnDefinition = "boolean[]")
-    private List<Boolean> sslStripping;
+    private List<Boolean> sslStripping = new ArrayList<>();
 
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     @Type(ListArrayType.class)
     @Column(name = "redirect_chain", columnDefinition = "text[]")
-    private List<String> redirectChain;
+    private List<String> redirectChain = new ArrayList<>();
+
+    @Column(name = "hostname_embedding")
+    private Integer hostnameEmbedding = 0;
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name = "javascript_check")
+    private String javascriptCheck = "";
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name = "shortening_service")
+    private String shorteningService = "";
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name = "has_ip_address")
+    private String hasIpAddress = "";
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Type(ListArrayType.class)
+    @Column(name = "tracking_descriptions", columnDefinition = "text[]")
+    private List<String> trackingDescriptions = new ArrayList<>();
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name = "url_encoding")
+    private String urlEncoding = "";
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name="has_executable")
+    private String hasExecutable = "";
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name = "dns_error")
+    private String dnsError = "";
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Column(name="ssl_error")
+    private String sslError = "";
+
+    // Custom getter for hostnameEmbedding
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public Integer getHostnameEmbedding() {
+        return hostnameEmbedding == 0 ? null : hostnameEmbedding;
+    }
+    // Custom getter for path
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public String getPath() {
+        return path == null || path.isEmpty() ? null : path;
+    }
+
+    // Custom getter for query
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @JsonProperty
+    public String getQuery() {
+        return query == null || query.equals("{}") ? null : query;
+    }
 }

src/main/java/com/safeqr/app/qrcode/entity/WifiEntity.java

@@ -2,6 +2,7 @@ package com.safeqr.app.qrcode.entity;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import jakarta.persistence.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -31,6 +32,7 @@ public class WifiEntity {
 
     private String ssid;
     private String password;
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private String encryption;
     private boolean hidden;
 }

src/main/java/com/safeqr/app/qrcode/model/EmailModel.java

@@ -35,4 +35,8 @@ public final class EmailModel extends QRCodeModel<EmailEntity> {
     public EmailEntity getDetails () {
         return emailVerificationService.getEmailEntityByQRCodeId(data.getId());
     }
+    @Override
+    public String retrieveClassification() {
+        return "";
+    }
 }

src/main/java/com/safeqr/app/qrcode/model/PhoneModel.java

@@ -34,4 +34,8 @@ public final class PhoneModel extends QRCodeModel<PhoneEntity> {
     public PhoneEntity getDetails () {
         return phoneVerificationService.getPhoneEntityByQRCodeId(data.getId());
     }
+    @Override
+    public String retrieveClassification() {
+        return "";
+    }
 }

src/main/java/com/safeqr/app/qrcode/model/QRCodeModel.java

@@ -10,4 +10,5 @@ public abstract class QRCodeModel<T>{
 
     public abstract void setDetails();
     public abstract T getDetails();
+    public abstract String retrieveClassification();
 }

src/main/java/com/safeqr/app/qrcode/model/SMSModel.java

@@ -34,4 +34,8 @@ public final class SMSModel extends QRCodeModel<SMSEntity> {
     public SMSEntity getDetails () {
         return smsVerificationService.getSMSEntityByQRCodeId(data.getId());
     }
+    @Override
+    public String retrieveClassification() {
+        return "";
+    }
 }

src/main/java/com/safeqr/app/qrcode/model/TextModel.java

@@ -34,4 +34,9 @@ public final class TextModel extends QRCodeModel<TextEntity> {
     public TextEntity getDetails () {
         return textVerificationService.getTextEntityByQRCodeId(data.getId());
     }
+
+    @Override
+    public String retrieveClassification() {
+        return "";
+    }
 }

src/main/java/com/safeqr/app/qrcode/model/URLModel.java

@@ -1,7 +1,6 @@
 package com.safeqr.app.qrcode.model;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.safeqr.app.qrcode.entity.EmailEntity;
 import com.safeqr.app.qrcode.entity.QRCodeEntity;
 import com.safeqr.app.qrcode.entity.URLEntity;
 import com.safeqr.app.qrcode.service.URLVerificationService;
@@ -12,8 +11,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
-
 @EqualsAndHashCode(callSuper = true)
 @Data
 public final class URLModel extends QRCodeModel<URLEntity> {
@@ -47,4 +44,9 @@ public final class URLModel extends QRCodeModel<URLEntity> {
     public URLEntity getDetails () {
         return urlVerificationService.getURLEntityByQRCodeId(data.getId());
     }
+
+    @Override
+    public String retrieveClassification() {
+        return "";
+    }
 }

src/main/java/com/safeqr/app/qrcode/model/WifiModel.java

@@ -26,6 +26,10 @@ public final class WifiModel extends QRCodeModel<WifiEntity> {
     @Override
     public void setDetails() {
         details = WifiEntity.builder().qrCodeId(data.getId()).build();
+
+        // Parse wifi string
+        wifiVerificationService.parseWifiString(details, data.getContents());
+
         // Insert into wifi table
         wifiVerificationService.insertDB(details);
     }
@@ -33,4 +37,9 @@ public final class WifiModel extends QRCodeModel<WifiEntity> {
     public WifiEntity getDetails () {
         return wifiVerificationService.getWifiEntityByQRCodeId(data.getId());
     }
+
+    @Override
+    public String retrieveClassification() {
+        return wifiVerificationService.getClassification(details.getEncryption());
+    }
 }

src/main/java/com/safeqr/app/qrcode/repository/URLRepository.java

@@ -1,10 +1,12 @@
 package com.safeqr.app.qrcode.repository;
 
 import com.safeqr.app.qrcode.entity.URLEntity;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.Optional;
 import java.util.UUID;
 
 public interface URLRepository extends GenericRepository<URLEntity> {
+    @Transactional
     Optional<URLEntity> findByQrCodeId(UUID qrCodeId);
 }

src/main/java/com/safeqr/app/qrcode/service/QRCodeTypeService.java

@@ -68,13 +68,18 @@ public class QRCodeTypeService {
     }
     // Get scanned qrcode details
     public BaseScanResponse getScannedQRCodeDetails(UUID qrCodeId){
+        return BaseScanResponse.builder().qrcode(getScannedQRCodeDetailsInModel(qrCodeId)).build();
+    }
+
+    public QRCodeModel<?> getScannedQRCodeDetailsInModel(UUID qrCodeId){
         // Find scanned qr code in qr code table
         QRCodeEntity qrCodeEntity = qrCodeRepository.findById(qrCodeId)
                 .orElseThrow(() -> new ResourceNotFoundExceptions("QR Code not found with id: " + qrCodeId));
         logger.info("qrCodeEntity: {}", qrCodeEntity);
         QRCodeModel<?> qrCodeModel = qrCodeFactoryProvider.createQRCodeInstance(qrCodeEntity);
         logger.info("Retrieved details: {}", qrCodeModel.getDetails());
-        return BaseScanResponse.builder().qrcode(qrCodeModel).build();
+
+        return qrCodeModel;
     }
 
     // Process Scanned QR Code
@@ -106,9 +111,35 @@ public class QRCodeTypeService {
         // Create the QR Code Instance based on the QR Code Type & insert into the respective table
         QRCodeModel<?> qrCodeModel = qrCodeFactoryProvider.createQRCodeInstance(scannedQR);
         qrCodeModel.setDetails();
+        // Get classifications based on verificationsv
+        scannedQR.setResult(qrCodeModel.retrieveClassification());
+        // update result category in qrcode table
+        qrCodeRepository.save(scannedQR);
 
         return BaseScanResponse.builder().qrcode(qrCodeModel).build();
     }
+
+    @Transactional
+    public QRCodeModel<?> scanGmailDecodedContents(String userId, String data) {
+        logger.info("Scan Gmail content: userId={}, data={}", userId, data);
+
+        // Get the QR Code Type
+        QRCodeTypeEntity qrType = getQRCodeType(data);
+
+        // Insert the QR Code into main qrcode table
+        QRCodeEntity scannedQR = qrCodeRepository.save(QRCodeEntity.builder()
+                .userId(userId)
+                .contents(data)
+                .info(qrType)
+                .createdAt(LocalDateTime.now())
+                .build());
+
+        // Create the QR Code Instance based on the QR Code Type & insert into the respective table
+        QRCodeModel<?> qrCodeModel = qrCodeFactoryProvider.createQRCodeInstance(scannedQR);
+        qrCodeModel.setDetails();
+
+        return qrCodeModel;
+    }
     // Returns Default type as text if it does not fit into any of the category
     private QRCodeTypeEntity getQRCodeType(String data) {
         return configs.stream()

src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java

@@ -2,7 +2,6 @@ package com.safeqr.app.qrcode.service;
 
 import static com.safeqr.app.constants.CommonConstants.*;
 
-import com.safeqr.app.exceptions.ResourceNotFoundExceptions;
 import com.safeqr.app.qrcode.dto.request.QRCodePayload;
 import com.safeqr.app.qrcode.dto.URLVerificationResponse;
 import com.safeqr.app.qrcode.entity.URLEntity;
@@ -13,14 +12,21 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLHandshakeException;
 import java.io.IOException;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 
 @Service
 public class URLVerificationService {
+    private static final int CONNECTION_TIMEOUT_MS = 10000;
+    private static final int READ_TIMEOUT_MS = 10000;
     private static final Logger logger = LoggerFactory.getLogger(URLVerificationService.class);
     private final URLRepository urlRepository;
     @Autowired
@@ -28,65 +34,262 @@ public class URLVerificationService {
         this.urlRepository = urlRepository;
     }
 
+    // Regular expression pattern for shortening services
+    private static final String SHORTENING_PATTERN =
+            "bit\\.ly|goo\\.gl|shorte\\.st|go2l\\.ink|x\\.co|ow\\.ly|t\\.co|tinyurl|tr\\.im|is\\.gd|cli\\.gs|" +
+                    "yfrog\\.com|migre\\.me|ff\\.im|tiny\\.cc|url4\\.eu|twit\\.ac|su\\.pr|twurl\\.nl|snipurl\\.com|" +
+                    "short\\.to|BudURL\\.com|ping\\.fm|post\\.ly|Just\\.as|bkite\\.com|snipr\\.com|fic\\.kr|loopt\\.us|" +
+                    "doiop\\.com|short\\.ie|kl\\.am|wp\\.me|rubyurl\\.com|om\\.ly|to\\.ly|bit\\.do|t\\.co|lnkd\\.in|" +
+                    "db\\.tt|qr\\.ae|adf\\.ly|goo\\.gl|bitly\\.com|cur\\.lv|tinyurl\\.com|ow\\.ly|bit\\.ly|ity\\.im|" +
+                    "q\\.gs|is\\.gd|po\\.st|bc\\.vc|twitthis\\.com|u\\.to|j\\.mp|buzurl\\.com|cutt\\.us|u\\.bb|yourls\\.org|" +
+                    "x\\.co|prettylinkpro\\.com|scrnch\\.me|filoops\\.info|vzturl\\.com|qr\\.net|1url\\.com|tweez\\.me|v\\.gd|" +
+                    "tr\\.im|link\\.zip\\.net";
+
+    // Regular expression pattern to match various IP address formats
+    private static final String IP_PATTERN =
+            "(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+                    "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\/)|" +
+                    "(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+                    "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\/)|" +
+                    "((0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\.(0x[0-9a-fA-F]{1,2})\\/)" +
+                    "(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4}|" +
+                    "([0-9]+(?:\\.[0-9]+){3}:[0-9]+)|" +
+                    "((?:(?:\\d|[01]?\\d\\d|2[0-4]\\d|25[0-5])\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d|\\d)(?:\\/\\d{1,2})?)";
+
+    // Define a Set of suspicious file extensions
+    private static final Set<String> SUSPICIOUS_EXTENSIONS = Stream.of(
+            ".exe", ".bat", ".sh", ".cmd", ".scr", ".pif", ".application", ".gadget",
+            ".vb", ".vbs", ".js", ".jse", ".ws", ".wsf", ".msc", ".com", ".cpl",
+            ".msi", ".ps1", ".py", ".pyc", ".pyo", ".rb", ".app", ".bin", ".run"
+    ).collect(Collectors.toUnmodifiableSet());
+
+    // Checks if the URL has executable file
+    public String hasExecutableFile(String urlPath) {
+        return Stream.of(urlPath)
+                .map(String::toLowerCase)
+                .map(path -> {
+                    int lastDotIndex = path.lastIndexOf('.');
+                    if (lastDotIndex != -1) {
+                        return path.substring(lastDotIndex);
+                    }
+                    return path.contains(".") || path.endsWith("/") ? null : "";
+                })
+                .filter(Objects::nonNull)
+                .map(extension -> SUSPICIOUS_EXTENSIONS.contains(extension) || extension.isEmpty() ? "Yes" : "")
+                .findFirst()
+                .orElse("");
+    }
+
     public URLEntity getURLEntityByQRCodeId(UUID qrCodeId) {
         logger.info("qrCodeId retrieving: {}", qrCodeId);
-        return urlRepository.findByQrCodeId(qrCodeId)
+//        return urlRepository.findByQrCodeId(qrCodeId)
-                .orElseThrow(() -> new ResourceNotFoundExceptions("URL not found for QR Code id: " + qrCodeId));
+//                .orElseThrow(() -> new ResourceNotFoundExceptions("URL not found for QR Code id: " + qrCodeId));
+        return urlRepository.findByQrCodeId(qrCodeId).orElse(null);
     }
 
     public void insertDB(URLEntity urlEntity) {
         urlRepository.save(urlEntity);
     }
     // Function to breakdown URL into subdomain, domain, topLevelDomain, query params, fragment
-    public URLEntity breakdownURL(String urlString) throws MalformedURLException {
+    public URLEntity breakdownURL(String urlString) {
         URLEntity urlObj = new URLEntity();
         try {
-            // Ensure the URL is properly encoded
+            //URL url = new URI(encodeUrl(urlString)).toURL();
-            String encodedUrl = encodeUrl(urlString);
+            URL url = new URI(urlString.replace(" ", "")).toURL();
-            URI uri = new URI(encodedUrl);
+            // Check for URL encoding in path and query
-            URL url = uri.toURL();
+            String query = parseQueryParams(url.getQuery());
+            String pathEncoding = checkURLEncoding(url.getPath());
+            String queryEncoding = query != null ? checkURLEncoding(query) : "";
 
+            // Combine encoding results
+            urlObj.setUrlEncoding(pathEncoding.equals("Yes") || queryEncoding.equals("Yes") ? "Yes" : "");
+
+            // encode url before proceeding the rest of the checks
+            url = new URI(encodeUrl(urlString)).toURL();
             String host = url.getHost();
-            // split host into subdomain, domain, topLevelDomain
+            populateHostDetails(host, urlObj);
-            String[] hostParts = host.split("\\.");
-            String subdomain = "";
 
-            if (hostParts.length >= 2) {
+            // Check for deceptive URL
-                // set topLevelDomain to the last part of the host
+            urlObj.setHostnameEmbedding(checkDeceptiveUrl(url));
-                urlObj.setTopLevelDomain(hostParts[hostParts.length - 1]);
-                // set domain to the second last part of the host
-                urlObj.setDomain(hostParts[hostParts.length - 2]);
-                // set subdomain to the first part of the host
-                if (hostParts.length > 2) {
-                    subdomain = String.join(".", java.util.Arrays.copyOfRange(hostParts, 0, hostParts.length - 2));
-                }
-            }
-            // set subdomain to URL host
-            urlObj.setSubdomain(subdomain);
 
-            String path = url.getPath();
+            // Check for Javascript code in url
-            //set path to URL path if it's not empty, otherwise set it to root path
+            urlObj.setJavascriptCheck(checkForJavascriptCode(urlString));
-            urlObj.setPath(path.isEmpty() ? "/" : path);
 
-            String query = url.getQuery();
+            // Check for url shortener
-            Map<String, String> queryParams = new HashMap<>();
+            urlObj.setShorteningService(hasShorteningService(urlString));
-            if (query != null) {
+
-                // split query params into key value pairs
+            // Check for IP address
-                for (String param : query.split("&")) {
+            urlObj.setHasIpAddress(hasIPAddress(urlString));
-                    String[] pair = param.split("=");
+
-                    queryParams.put(pair[0], pair.length > 1 ? pair[1] : "");
+            // Check for suspicious file extensions
-                }
+            urlObj.setHasExecutable(hasExecutableFile(urlString));
-                logger.info("queryParams: {}", queryParams);
+
-            }
+            urlObj.setPath(Optional.ofNullable(url.getPath()).filter(p -> !p.isEmpty()).orElse(""));
-            // set query params to URL query
+
-            urlObj.setQuery(queryParams.toString());
+            urlObj.setQuery(parseQueryParams(url.getQuery()));
-            // set fragment to URL ref
             urlObj.setFragment(Optional.ofNullable(url.getRef()).orElse(""));
-        } catch (URISyntaxException | MalformedURLException e) {
+
+            // Check for tracking parameters
+            urlObj.setTrackingDescriptions(getTrackingDescriptions(url.getQuery()));
+
+        } catch (Exception e) {
             logger.error("Error in breaking down URL: {}", e.getMessage());
+            e.printStackTrace();
         }
         return urlObj;
     }
+
+    private void populateHostDetails(String host, URLEntity urlObj) {
+        logger.info("Host: {}", host);
+
+        if (host != null && !host.isEmpty()) {
+            if (isIpAddress(host)) {
+                // Handle IP address
+                urlObj.setDomain(host);
+                urlObj.setTopLevelDomain("");  // No TLD for IP addresses
+                urlObj.setSubdomain("");       // No subdomain for IP addresses
+            } else {
+                // Handle regular domain name
+                String[] hostParts = host.split("\\.");
+
+                int length = hostParts.length;
+
+                if (length >= 2) {
+                    urlObj.setTopLevelDomain(hostParts[length - 1]); // TLD, e.g., "com"
+                    urlObj.setDomain(hostParts[length - 2]);         // Domain, e.g., "example"
+                    urlObj.setSubdomain(length > 2 ? String.join(".", Arrays.copyOfRange(hostParts, 0, length - 2)) : "");
+                } else if (length == 1) {
+                    // Handle cases like 'localhost' where there's no TLD
+                    urlObj.setDomain(hostParts[0]);
+                    urlObj.setTopLevelDomain("");  // No TLD
+                    urlObj.setSubdomain("");       // No subdomain
+                }
+            }
+        }
+    }
+    // List of common tracking parameters with their descriptions
+    private static final Map<String, String> TRACKING_DESCRIPTIONS = Map.ofEntries(
+            Map.entry("utm_source", "Campaign Source: Identifies which site sent the traffic."),
+            Map.entry("utm_medium", "Campaign Medium: Identifies what type of link was used."),
+            Map.entry("utm_campaign", "Campaign Name: Identifies a specific product promotion or campaign."),
+            Map.entry("utm_term", "Campaign Term: Identifies search terms."),
+            Map.entry("utm_content", "Campaign Content: Differentiates similar content or links within the same ad."),
+            Map.entry("gclid", "Google Click Identifier: Used by Google Ads to track clicks."),
+            Map.entry("fbclid", "Facebook Click Identifier: Used by Facebook to track clicks."),
+            Map.entry("tracking_id", "Tracking ID: General identifier for tracking purposes."),
+            Map.entry("affiliate_id", "Affiliate ID: Identifies traffic from affiliates."),
+            Map.entry("ref", "Referrer: Identifies the referrer site."),
+            Map.entry("referrer", "Referrer: Identifies the referrer site.")
+    );
+
+    // Regex pattern to capture key-value pairs in the query string
+    private static final Pattern PARAM_PATTERN = Pattern.compile(
+            "(?<key>[^=&]+)=(?<value>[^&]+)",
+            Pattern.CASE_INSENSITIVE
+    );
+
+    // Static method to detect and return tracking parameter descriptions in a URL
+    private List<String> getTrackingDescriptions(String query) {
+        if (query == null || query.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        Matcher matcher = PARAM_PATTERN.matcher(query);
+        List<String> foundDescriptions = new ArrayList<>();
+
+        while (matcher.find()) {
+            String key = matcher.group("key").toLowerCase();
+            String value = URLDecoder.decode(matcher.group("value"), StandardCharsets.UTF_8);
+            if (TRACKING_DESCRIPTIONS.containsKey(key)) {
+                foundDescriptions.add(TRACKING_DESCRIPTIONS.get(key) + ": " + value);
+            }
+        }
+
+        return foundDescriptions;
+    }
+
+    private int checkDeceptiveUrl(URL url) {
+        String[] parts = url.getHost().split("\\.");
+        if (parts.length < 3) return 0;
+
+        Set<String> commonTlds = new HashSet<>(Arrays.asList("com", "org", "net", "edu", "gov"));
+
+        for (int i = parts.length - 2; i >= 1; i--) {
+            if (commonTlds.contains(parts[i]) && !commonTlds.contains(parts[i - 1]) && i != parts.length - 2) {
+                logger.warn("Potentially deceptive URL detected: {} (Suspicious domain: {}.{})",
+                        url, parts[i - 1], parts[i]);
+                return 1;
+            }
+        }
+        return 0;
+    }
+
+    private String checkForJavascriptCode(String url) {
+        // Decode the URL
+        String decodedUrl = URLDecoder.decode(url, StandardCharsets.UTF_8);
+
+        // Patterns to detect 'javascript:', '<script>', and 'on*=' attributes
+        List<Pattern> maliciousPatterns = Arrays.asList(
+                Pattern.compile("javascript:", Pattern.CASE_INSENSITIVE),
+                Pattern.compile("<\\s*script", Pattern.CASE_INSENSITIVE),
+                Pattern.compile("on(click|mouseover|load|error|unload|submit|reset|focus|blur|change|select|keydown|keyup|keypress|mousedown|mousemove|mouseup|mouseenter|mouseleave|contextmenu|dblclick)\\s*=", Pattern.CASE_INSENSITIVE)
+        );
+
+        // Check for any malicious pattern in the URL
+        for (Pattern pattern : maliciousPatterns) {
+            Matcher matcher = pattern.matcher(decodedUrl);
+            if (matcher.find()) {
+                return "Javascript found in URL.";
+            }
+        }
+        return "";
+    }
+
+    // Function to detect if the URL uses a shortening service
+    private String hasShorteningService(String url) {
+        Pattern pattern = Pattern.compile(SHORTENING_PATTERN, Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(url);
+        return matcher.find() ? "Yes" : "";
+    }
+
+    // Function to check text encoding in a URL
+    private static String checkURLEncoding(String pathTextPart) {
+        // Decode the text
+        String decodedText = URLDecoder.decode(pathTextPart, StandardCharsets.UTF_8);
+
+        // Check if the decoded text matches the original text
+        return decodedText.equals(pathTextPart) ? "" : "Yes";
+    }
+
+    // Function to detect if the URL has an IP address
+    private static String hasIPAddress(String url) {
+        Pattern pattern = Pattern.compile(IP_PATTERN, Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(url);
+        return matcher.find() ? "URL contains IP address." : "";
+    }
+    // Check if the host is an IP address
+    private boolean isIpAddress(String host) {
+        // Regex to match IPv4 addresses
+        String ipv4Pattern = "\\d+\\.\\d+\\.\\d+\\.\\d+";
+        // Regex to match IPv6 addresses
+        String ipv6Pattern = "([a-fA-F0-9:]+:+)+[a-fA-F0-9]+";
+
+        return host.matches(ipv4Pattern) || host.matches(ipv6Pattern);
+    }
+    private String parseQueryParams(String query) {
+        if (query == null || query.isEmpty()) return "{}";
+        Map<String, String> queryParams = new HashMap<>();
+        for (String param : query.split("&")) {
+            String[] pair = param.split("=", 2);
+            String key = pair[0];
+            String value = pair.length > 1 ? pair[1] : "";
+            if (!key.isEmpty()) {
+                queryParams.put(key, value);
+            }
+        }
+        return queryParams.toString();
+    }
+
     private String encodeUrl(String urlString) throws MalformedURLException {
         try {
             URL url = new URL(urlString);
@@ -119,7 +322,7 @@ public class URLVerificationService {
 
     public void countAndTrackRedirects(String urlString, URLEntity details) throws IOException {
         try {
-            URI uri = new URI(urlString);
+            URI uri = new URI(encodeUrl(urlString));
             URL url = uri.toURL();
             List<String> redirectChain = new ArrayList<>();
             List<String> hstsHeaderList = new ArrayList<>();
@@ -143,6 +346,8 @@ public class URLVerificationService {
                 HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                 connection.setRequestMethod("GET");
                 connection.setInstanceFollowRedirects(false);
+                connection.setConnectTimeout(CONNECTION_TIMEOUT_MS);
+                connection.setReadTimeout(READ_TIMEOUT_MS);
 
                 int responseCode = connection.getResponseCode();
                 redirected = (responseCode >= 300 && responseCode < 400);
@@ -181,6 +386,23 @@ public class URLVerificationService {
             details.setHstsHeader(hstsHeaderList);
         } catch (URISyntaxException e){
             logger.error("Error in breaking down URL: {}", e.getMessage());
+        } catch (SSLHandshakeException e) {
+            logger.error("SSL Handshake Exception: {}", e.getMessage());
+            details.setSslError("SSL Handshake Exception: " + e.getMessage());
+        } catch (SocketTimeoutException e) {
+            logger.error("Connection timed out: {}", e.getMessage());
+            details.setDnsError("Connection timed out: " + e.getMessage());
+        } catch (UnknownHostException e) {
+            logger.error("Unknown Host Exception: {}", e.getMessage());
+            details.setDnsError("Unknown Host Exception: " + e.getMessage());
+        } catch (NoRouteToHostException e) {
+            details.setDnsError("Error: " + e.getMessage());
+        } catch (ConnectException e) {
+            details.setDnsError("Connection Error: " + e.getMessage());
+        } catch (SocketException e) {
+            details.setDnsError("Socket Error: " + e.getMessage());
+        } catch (Exception e) {
+            details.setDnsError("Exception: " + e.getMessage());
         }
     }
     // Function to check if the redirect is from HTTPS to HTTP

src/main/java/com/safeqr/app/qrcode/service/WifiVerificationService.java

@@ -10,6 +10,8 @@ import org.springframework.stereotype.Service;
 
 import java.util.UUID;
 
+import static com.safeqr.app.constants.CommonConstants.*;
+
 @Service
 public class WifiVerificationService {
     private final WifiRepository wifiRepository;
@@ -28,4 +30,51 @@ public class WifiVerificationService {
         wifiRepository.save(wifiEntity);
     }
 
+    public void parseWifiString(WifiEntity wifiEntity, String wifiString) {
+        wifiString = wifiString.substring(5);
+        // Split the string by semicolons
+        String[] parts = wifiString.split(";");
+
+        for (String part : parts) {
+            if (part.startsWith("T:")) {
+                wifiEntity.setEncryption(part.substring(2));
+            } else if (part.startsWith("S:")) {
+                wifiEntity.setSsid(part.substring(2));
+            } else if (part.startsWith("P:")) {
+                wifiEntity.setPassword(part.substring(2));
+            } else if (part.startsWith("H:")) {
+                wifiEntity.setHidden(Boolean.parseBoolean(part.substring(2)));
+            }
+        }
+
+        // Unescape special characters in SSID and password
+        wifiEntity.setSsid(unescapeString(wifiEntity.getSsid()));
+        wifiEntity.setPassword(unescapeString(wifiEntity.getPassword()));
+    }
+
+    private String unescapeString(String input) {
+        return input.replace("\\:", ":")
+                .replace("\\;", ";")
+                .replace("\\,", ",")
+                .replace("\\\\", "\\");
+    }
+
+    public String getClassification(String encryptionType) {
+        // Check if encryptionType is null
+        if (encryptionType == null) {
+            return CLASSIFY_UNSAFE;
+        }
+
+        if (encryptionType.equalsIgnoreCase("WPA") ||
+                encryptionType.equalsIgnoreCase("WPA2") ||
+                encryptionType.equalsIgnoreCase("WPA3")) {
+            return CLASSIFY_SAFE;
+        } else if (encryptionType.equalsIgnoreCase("WEP")) {
+            return CLASSIFY_WARNING;
+        } else if (encryptionType.equalsIgnoreCase("nopass")) {
+            return CLASSIFY_UNSAFE;
+        } else {
+            return CLASSIFY_UNKNOWN;
+        }
+    }
 }

src/main/java/com/safeqr/app/spark/model/URLFeatures.java

@@ -0,0 +1,134 @@
+package com.safeqr.app.spark.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class URLFeatures {
+    private Long domain;
+    private Long subdomain;
+    private Long topLevelDomain;
+    private Long query;
+    private Long fragment;
+    private Long redirect;
+    private Long path;
+    private Long redirectChain;
+    private Long hstsHeader;
+    private Long sslStripping;
+    private Long hostnameEmbedding;
+    private Long javascriptCheck;
+    private Long shorteningService;
+    private Long hasIpAddress;
+    private Long trackingDescriptions;
+    private Long urlEncoding;
+    private Long hasExecutable;
+    private Long tls;
+    private Long contents;
+    private String target; // This is the label, may be null if predicting
+
+    // Custom setter for tls (qr_code_type_id)
+    public void setTls(Long tls) {
+        if (tls != null) {
+            this.tls = tls == 1 ? 0 : tls == 9 ? 1 : tls;
+        } else {
+            this.tls = 0L;
+        }
+    }
+
+    // Custom setter for hostnameEmbedding and other similar columns
+    public void setHostnameEmbedding(Long hostnameEmbedding) {
+        this.hostnameEmbedding = (hostnameEmbedding != null && hostnameEmbedding != 0) ? 1L : 0L;
+    }
+
+    public void setJavascriptCheck(Long javascriptCheck) {
+        this.javascriptCheck = (javascriptCheck != null && javascriptCheck != 0) ? 1L : 0L;
+    }
+
+    public void setShorteningService(Long shorteningService) {
+        this.shorteningService = (shorteningService != null && shorteningService != 0) ? 1L : 0L;
+    }
+
+    public void setHasIpAddress(Long hasIpAddress) {
+        this.hasIpAddress = (hasIpAddress != null && hasIpAddress != 0) ? 1L : 0L;
+    }
+
+    public void setUrlEncoding(Long urlEncoding) {
+        this.urlEncoding = (urlEncoding != null && urlEncoding != 0) ? 1L : 0L;
+    }
+
+    public void setHasExecutable(Long hasExecutable) {
+        this.hasExecutable = (hasExecutable != null && hasExecutable != 0) ? 1L : 0L;
+    }
+
+    public void setTrackingDescriptions(Long trackingDescriptions) {
+        this.trackingDescriptions = (trackingDescriptions != null && trackingDescriptions != 0) ? 1L : 0L;
+    }
+
+    // Custom setter for sslStripping
+    public void setSslStripping(String sslStripping) {
+        if (sslStripping != null && "true".equalsIgnoreCase(sslStripping)) {
+            this.sslStripping = 1L;
+        } else {
+            this.sslStripping = 0L;
+        }
+    }
+
+    // Custom setter for hstsHeader
+    public void setHstsHeader(String hstsHeader) {
+        if (hstsHeader == null || "0".equals(hstsHeader)) {
+            this.hstsHeader = 0L;
+        } else if (hstsHeader.startsWith("{") && hstsHeader.endsWith("}")) {
+            Pattern pattern = Pattern.compile("\"(.*?)\"");
+            Matcher matcher = pattern.matcher(hstsHeader);
+            if (matcher.find() && matcher.group(1).toLowerCase().contains("no")) {
+                this.hstsHeader = 0L;
+            } else {
+                this.hstsHeader = 1L;
+            }
+        } else {
+            this.hstsHeader = 0L;
+        }
+    }
+
+    // Custom setters for calculating string lengths
+    public void setDomain(String domain) {
+        this.domain = (domain != null) ? (long) domain.length() : 0L;
+    }
+
+    public void setSubdomain(String subdomain) {
+        this.subdomain = (subdomain != null) ? (long) subdomain.length() : 0L;
+    }
+
+    public void setTopLevelDomain(String topLevelDomain) {
+        this.topLevelDomain = (topLevelDomain != null) ? (long) topLevelDomain.length() : 0L;
+    }
+
+    public void setQuery(String query) {
+        this.query = (query != null) ? (long) query.length() : 0L;
+    }
+
+    public void setFragment(String fragment) {
+        this.fragment = (fragment != null) ? (long) fragment.length() : 0L;
+    }
+
+    public void setPath(String path) {
+        this.path = (path != null) ? (long) path.length() : 0L;
+    }
+
+    public void setRedirectChain(String redirectChain) {
+        this.redirectChain = (redirectChain != null) ? (long) redirectChain.length() : 0L;
+    }
+
+    public void setContents(String contents) {
+        this.contents = (contents != null) ? (long) contents.length() : 0L;
+    }
+}

src/main/java/com/safeqr/app/user/controller/UserController.java

@@ -37,49 +37,49 @@ public class UserController {
 
     @GetMapping(value = API_URL_USER_GET, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<UserResponseDto> getUser(@RequestHeader(name = HEADER_USER_ID) String userId) {
-        logger.info("Invoking GET User endpoint");
+        logger.info("User Id Invoking GET User endpoint: {}", userId);
         return ResponseEntity.ok(userService.getUserById(userId));
     }
 
     @GetMapping(value = API_URL_USER_GET_SCANNED_HISTORIES, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<List<ScannedHistoriesDto>> getUserScannedHistories(@RequestHeader(name = HEADER_USER_ID) String userId) {
-        logger.info("Invoking GET User Scanned Histories endpoint");
+        logger.info("User Id Invoking GET User Scanned Histories endpoint: {}", userId);
         return ResponseEntity.ok(userService.getUserScannedHistories(userId));
     }
 
     @PutMapping(value = API_URL_USER_DELETE_SCANNED_HISTORIES, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<BaseResponse> deleteScannedHistory(@RequestHeader(name = HEADER_USER_ID) String userId, @RequestBody BookmarkRequestDto bookmarkRequestDto) {
-        logger.info("Invoking PUT Delete Single Scanned History endpoint");
+        logger.info("User Id Invoking PUT Delete Single Scanned History endpoint: {}", userId);
         return ResponseEntity.ok(userService.deleteScannedHistory(userId, bookmarkRequestDto.getQrCodeId()));
     }
 
     @PutMapping(value = API_URL_USER_DELETE_ALL_SCANNED_HISTORIES, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<BaseResponse> deleteAllScannedHistories(@RequestHeader(name = HEADER_USER_ID) String userId) {
-        logger.info("Invoking PUT Delete All Scanned Histories endpoint");
+        logger.info("User Id Invoking PUT Delete All Scanned Histories endpoint: {}", userId);
         return ResponseEntity.ok(userService.deleteAllScannedHistoriesByUserId(userId));
     }
 
     @GetMapping(value = API_URL_USER_GET_BOOKMARKS, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<List<ScannedHistoriesDto>> getUserBookmarks(@RequestHeader(name = HEADER_USER_ID) String userId) {
-        logger.info("Invoking GET User bookmarks endpoint");
+        logger.info("User Id Invoking GET User bookmarks endpoint: {}", userId);
         return ResponseEntity.ok(userService.getUserBookmarks(userId));
     }
 
     @PostMapping(value = API_URL_USER_SET_BOOKMARK, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<BaseResponse> setBookmark(@RequestHeader(name = HEADER_USER_ID) String userId, @RequestBody BookmarkRequestDto bookmarkRequestDto) {
-        logger.info("Invoking POST User bookmark endpoint");
+        logger.info("User Id Invoking POST User bookmark endpoint: {}", userId);
         return ResponseEntity.ok(userService.setBookmark(userId, bookmarkRequestDto.getQrCodeId()));
     }
 
     @PutMapping(value = API_URL_USER_DELETE_BOOKMARK, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<BaseResponse> deleteBookmark(@RequestHeader(name = HEADER_USER_ID) String userId, @RequestBody BookmarkRequestDto bookmarkRequestDto) {
-        logger.info("Invoking PUT Delete Single Bookmark endpoint");
+        logger.info("User Id Invoking PUT Delete Single Bookmark endpoint: {}", userId);
         return ResponseEntity.ok(userService.deleteBookmark(userId, bookmarkRequestDto.getQrCodeId()));
     }
 
     @PutMapping(value = API_URL_USER_DELETE_ALL_BOOKMARK, produces = MediaType.APPLICATION_JSON_VALUE)
     public ResponseEntity<BaseResponse> deleteAllBookmark(@RequestHeader(name = HEADER_USER_ID) String userId) {
-        logger.info("Invoking PUT Delete All Bookmark endpoint");
+        logger.info("User Id Invoking PUT Delete All Bookmark endpoint: {}", userId);
         return ResponseEntity.ok(userService.deleteAllBookmarkByUserId(userId));
     }
 }

src/main/java/com/safeqr/app/user/entity/UserEntity.java

@@ -33,4 +33,7 @@ public class UserEntity {
     @Column(name = "roles", columnDefinition = "text[]")
     private List<String> roles;
     private String status;
+
+    @Column(name = "gmail_history_id")
+    private Long gmailHistoryId;
 }

src/main/java/com/safeqr/app/utils/AsyncConfig.java

@@ -0,0 +1,34 @@
+package com.safeqr.app.utils;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.context.annotation.Bean;
+import java.util.concurrent.Executor;
+
+@Configuration
+@EnableAsync
+public class AsyncConfig {
+
+    @Bean(name = "taskExecutor")
+    public Executor taskExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+
+        // Sets the number of core threads. These threads are always kept alive.
+        executor.setCorePoolSize(2);
+
+        // Sets the maximum number of threads that can be created by the pool.
+        executor.setMaxPoolSize(2);
+
+        // Sets the size of the queue to hold tasks before they are executed.
+        executor.setQueueCapacity(500);
+
+        // Sets the prefix for the names of the threads created by this pool.
+        executor.setThreadNamePrefix("GmailProcessing-");
+
+        // Initializes the executor to apply the configuration and make it ready to use.
+        executor.initialize();
+
+        return executor;
+    }
+}

src/main/java/com/safeqr/app/utils/DateParsingUtils.java

@@ -0,0 +1,42 @@
+package com.safeqr.app.utils;
+
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.util.Locale;
+
+public class DateParsingUtils {
+
+    private DateParsingUtils() {
+        // private constructor to hide the implicit public one
+    }
+
+    private static final DateTimeFormatter INPUT_FORMATTER = new DateTimeFormatterBuilder()
+            .appendPattern("EEE, ")
+            .appendPattern("[ ]") // This makes a single space optional
+            .appendPattern("[ ]") // This allows for a second optional space
+            .appendValue(ChronoField.DAY_OF_MONTH, 1, 2, java.time.format.SignStyle.NOT_NEGATIVE)
+            .appendPattern(" MMM yyyy HH:mm:ss Z")
+            .toFormatter(Locale.ENGLISH);
+
+    private static final DateTimeFormatter OUTPUT_FORMATTER =
+            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS Z");
+
+    public static String parseAndFormatDate(String inputDate) {
+        try {
+            OffsetDateTime dateTime = OffsetDateTime.parse(inputDate, INPUT_FORMATTER);
+            return dateTime.format(OUTPUT_FORMATTER);
+        } catch (Exception e) {
+            throw new RuntimeException("Error parsing date: " + inputDate, e);
+        }
+    }
+
+    public static OffsetDateTime parseDate(String inputDate) {
+        try {
+            return OffsetDateTime.parse(inputDate, INPUT_FORMATTER);
+        } catch (Exception e) {
+            throw new RuntimeException("Error parsing date: " + inputDate, e);
+        }
+    }
+}

src/main/resources/cv_model/bestModel/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.PipelineModel","timestamp":1723422050490,"sparkVersion":"3.4.3","uid":"PipelineModel_4ecdd9f71524","paramMap":{"stageUids":["StringIndexer_d3c63289c493","VectorAssembler_517fc429fbfb","RandomForestClassifier_4909b7ca2bbe"]},"defaultParamMap":{}}

src/main/resources/cv_model/bestModel/stages/0_StringIndexer_d3c63289c493/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StringIndexerModel","timestamp":1723422050930,"sparkVersion":"3.4.3","uid":"StringIndexer_d3c63289c493","paramMap":{"outputCol":"indexed_target","inputCol":"target"},"defaultParamMap":{"stringOrderType":"frequencyDesc","handleInvalid":"error","outputCol":"StringIndexer_d3c63289c493__output"}}

src/main/resources/cv_model/bestModel/stages/1_VectorAssembler_517fc429fbfb/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1723422054241,"sparkVersion":"3.4.3","uid":"VectorAssembler_517fc429fbfb","paramMap":{"inputCols":["domain","subdomain","top_level_domain","query","fragment","redirect","path","redirect_chain","hsts_header","ssl_stripping","hostname_embedding","javascript_check","shortening_service","has_ip_address","tracking_descriptions","url_encoding","has_executable","tls","contents"],"outputCol":"features"},"defaultParamMap":{"handleInvalid":"error","outputCol":"VectorAssembler_517fc429fbfb__output"}}

src/main/resources/cv_model/bestModel/stages/2_RandomForestClassifier_4909b7ca2bbe/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.RandomForestClassificationModel","timestamp":1723422054559,"sparkVersion":"3.4.3","uid":"RandomForestClassifier_4909b7ca2bbe","paramMap":{"featuresCol":"features","numTrees":100,"maxDepth":10,"labelCol":"indexed_target"},"defaultParamMap":{"featuresCol":"features","rawPredictionCol":"rawPrediction","maxBins":32,"predictionCol":"prediction","minInstancesPerNode":1,"minWeightFractionPerNode":0.0,"cacheNodeIds":false,"minInfoGain":0.0,"numTrees":20,"maxDepth":5,"impurity":"gini","subsamplingRate":1.0,"leafCol":"","labelCol":"label","probabilityCol":"probability","bootstrap":true,"featureSubsetStrategy":"auto","checkpointInterval":10,"maxMemoryInMB":256,"seed":6182040365248539008},"numFeatures":19,"numClasses":4,"numTrees":100}

src/main/resources/cv_model/estimator/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.Pipeline","timestamp":1723422048367,"sparkVersion":"3.4.3","uid":"Pipeline_58a1fe22f286","paramMap":{"stageUids":["StringIndexer_d3c63289c493","VectorAssembler_517fc429fbfb","RandomForestClassifier_4909b7ca2bbe"]},"defaultParamMap":{}}

src/main/resources/cv_model/estimator/stages/0_StringIndexer_d3c63289c493/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StringIndexer","timestamp":1723422049018,"sparkVersion":"3.4.3","uid":"StringIndexer_d3c63289c493","paramMap":{"outputCol":"indexed_target","inputCol":"target"},"defaultParamMap":{"stringOrderType":"frequencyDesc","handleInvalid":"error","outputCol":"StringIndexer_d3c63289c493__output"}}

src/main/resources/cv_model/estimator/stages/1_VectorAssembler_517fc429fbfb/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1723422049580,"sparkVersion":"3.4.3","uid":"VectorAssembler_517fc429fbfb","paramMap":{"inputCols":["domain","subdomain","top_level_domain","query","fragment","redirect","path","redirect_chain","hsts_header","ssl_stripping","hostname_embedding","javascript_check","shortening_service","has_ip_address","tracking_descriptions","url_encoding","has_executable","tls","contents"],"outputCol":"features"},"defaultParamMap":{"handleInvalid":"error","outputCol":"VectorAssembler_517fc429fbfb__output"}}

src/main/resources/cv_model/estimator/stages/2_RandomForestClassifier_4909b7ca2bbe/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.RandomForestClassifier","timestamp":1723422050089,"sparkVersion":"3.4.3","uid":"RandomForestClassifier_4909b7ca2bbe","paramMap":{"featuresCol":"features","labelCol":"indexed_target"},"defaultParamMap":{"featuresCol":"features","rawPredictionCol":"rawPrediction","maxBins":32,"predictionCol":"prediction","minInstancesPerNode":1,"minWeightFractionPerNode":0.0,"cacheNodeIds":false,"minInfoGain":0.0,"numTrees":20,"maxDepth":5,"impurity":"gini","subsamplingRate":1.0,"leafCol":"","labelCol":"label","probabilityCol":"probability","bootstrap":true,"featureSubsetStrategy":"auto","checkpointInterval":10,"maxMemoryInMB":256,"seed":6182040365248539008}}

src/main/resources/cv_model/evaluator/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator","timestamp":1723422047764,"sparkVersion":"3.4.3","uid":"MulticlassClassificationEvaluator_f31cf4d2b0db","paramMap":{"metricName":"accuracy","labelCol":"indexed_target"},"defaultParamMap":{"eps":1.0E-15,"beta":1.0,"metricName":"f1","predictionCol":"prediction","labelCol":"label","metricLabel":0.0,"probabilityCol":"probability"}}

src/main/resources/cv_model/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.tuning.CrossValidatorModel","timestamp":1723422046660,"sparkVersion":"3.4.3","uid":"CrossValidatorModel_5c96b33c8d82","paramMap":{"seed":-2084793586583917283,"numFolds":5,"foldCol":"","estimatorParamMaps":[[{"parent":"RandomForestClassifier_4909b7ca2bbe","name":"numTrees","value":"100","isJson":"true"},{"parent":"RandomForestClassifier_4909b7ca2bbe","name":"maxDepth","value":"10","isJson":"true"}]]},"defaultParamMap":{"seed":880116102,"numFolds":3,"foldCol":""},"avgMetrics":[0.8736361548764979],"persistSubModels":false}