isky/backend-springboot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ml init

Browse Source
This commit is contained in:
ltiongku
2024-08-12 23:55:08 +08:00
parent ef1fb9f6e0
commit 53f9acd922
6 changed files with 124 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pom.xml
View File

@@ -121,26 +121,7 @@
<artifactId>jackson-annotations</artifactId> <artifactId>jackson-annotations</artifactId>
<version>2.17.2</version> <version>2.17.2</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.13</artifactId>
<version>3.4.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.13</artifactId>
<version>3.4.3</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-mllib -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.13</artifactId>
<version>3.4.3</version>
<scope>provided</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>

10
src/main/java/com/safeqr/app/qrcode/controller/QRCodeTypeController.java
View File

@@ -61,10 +61,12 @@ public class QRCodeTypeController {
return ResponseEntity.ok(qrCodeTypeService.detectType(payload).block()); return ResponseEntity.ok(qrCodeTypeService.detectType(payload).block());
} }
@PostMapping(API_URL_QRCODE_VERIFY_URL) @PostMapping(value = API_URL_QRCODE_VERIFY_URL, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<URLVerificationResponse> verifyURL(@RequestBody QRCodePayload payload) { public ResponseEntity<BaseScanResponse> verifyURL(@RequestBody QRCodePayload payload,
URLVerificationResponse response = urlVerificationService.verifyURL(payload); @RequestHeader(required = false, name = HEADER_USER_ID) String userId) {
return ResponseEntity.ok(response); logger.info("User Id Invoking verify url endpoint: {}", userId);
return ResponseEntity.ok(qrCodeTypeService.scanQRCode(userId, payload));
} }
@PostMapping(API_URL_QRCODE_VIRUS_TOTAL_CHECK) @PostMapping(API_URL_QRCODE_VIRUS_TOTAL_CHECK)

2
src/main/java/com/safeqr/app/qrcode/model/URLModel.java
View File

@@ -47,6 +47,6 @@ public final class URLModel extends QRCodeModel<URLEntity> {
@Override @Override
public String retrieveClassification() { public String retrieveClassification() {
return ""; return urlVerificationService.getClassification(this);
} }
} }

29
src/main/java/com/safeqr/app/qrcode/service/URLVerificationService.java
View File

@@ -1,11 +1,10 @@
package com.safeqr.app.qrcode.service; package com.safeqr.app.qrcode.service;
import static com.safeqr.app.constants.CommonConstants.*; import static com.safeqr.app.constants.CommonConstants.*;
import com.safeqr.app.qrcode.dto.request.QRCodePayload;
import com.safeqr.app.qrcode.dto.URLVerificationResponse;
import com.safeqr.app.qrcode.entity.URLEntity; import com.safeqr.app.qrcode.entity.URLEntity;
import com.safeqr.app.qrcode.model.URLModel;
import com.safeqr.app.qrcode.repository.URLRepository; import com.safeqr.app.qrcode.repository.URLRepository;
import com.safeqr.app.spark.service.MLModelService;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -29,9 +28,11 @@ public class URLVerificationService {
private static final int READ_TIMEOUT_MS = 10000; private static final int READ_TIMEOUT_MS = 10000;
private static final Logger logger = LoggerFactory.getLogger(URLVerificationService.class); private static final Logger logger = LoggerFactory.getLogger(URLVerificationService.class);
private final URLRepository urlRepository; private final URLRepository urlRepository;
private final MLModelService mlModelService;
@Autowired @Autowired
public URLVerificationService(URLRepository urlRepository) { public URLVerificationService(URLRepository urlRepository, MLModelService mlModelService) {
this.urlRepository = urlRepository; this.urlRepository = urlRepository;
this.mlModelService = mlModelService;
} }
// Regular expression pattern for shortening services // Regular expression pattern for shortening services
@@ -425,22 +426,8 @@ public class URLVerificationService {
return INFO_NON_SECURE_CONNECTION; return INFO_NON_SECURE_CONNECTION;
} }
public URLVerificationResponse verifyURL(QRCodePayload payload) { // Get Classification using ML Model
URLVerificationResponse response = new URLVerificationResponse(); public String getClassification(URLModel urlModel){
try { return mlModelService.predict(urlModel);
java.net.URL url = new java.net.URL(payload.getData());
String protocol = url.getProtocol();
if ("https".equalsIgnoreCase(protocol)) {
response.setSecure(true);
response.setMessage("The connection is secure.");
} else {
response.setSecure(false);
response.setMessage("The connection is not secure.");
}
} catch (Exception e) {
response.setSecure(false);
response.setMessage("Invalid URL.");
}
return response;
} }
} }

148
src/main/java/com/safeqr/app/spark/model/URLFeatures.java
View File

@@ -1,134 +1,158 @@
package com.safeqr.app.spark.model; package com.safeqr.app.spark.model;
import lombok.AllArgsConstructor; import com.safeqr.app.qrcode.entity.URLEntity;
import lombok.Builder; import com.safeqr.app.qrcode.model.URLModel;
import lombok.Data; import lombok.*;
import lombok.NoArgsConstructor;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@Data @Getter
@Builder @Builder
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor @AllArgsConstructor
public class URLFeatures { public class URLFeatures {
private Long domain; private Double domain;
private Long subdomain; private Double subdomain;
private Long topLevelDomain; private Double topLevelDomain;
private Long query; private Double query;
private Long fragment; private Double fragment;
private Long redirect; private Double redirect;
private Long path; private Double path;
private Long redirectChain; private Double redirectChain;
private Long hstsHeader; private Double hstsHeader;
private Long sslStripping; private Double sslStripping;
private Long hostnameEmbedding; private Double hostnameEmbedding;
private Long javascriptCheck; private Double javascriptCheck;
private Long shorteningService; private Double shorteningService;
private Long hasIpAddress; private Double hasIpAddress;
private Long trackingDescriptions; private Double trackingDescriptions;
private Long urlEncoding; private Double urlEncoding;
private Long hasExecutable; private Double hasExecutable;
private Long tls; private Double tls;
private Long contents; private Double contents;
private String target; // This is the label, may be null if predicting private String target; // This is the label, may be null if predicting
public static URLFeatures fromEntity(URLModel urlModel) {
URLFeatures features = URLFeatures.builder()
.build();
features.setDomain(urlModel.getDetails().getDomain());
features.setSubdomain(urlModel.getDetails().getSubdomain());
features.setTopLevelDomain(urlModel.getDetails().getTopLevelDomain());
features.setQuery(urlModel.getDetails().getQuery());
features.setFragment(urlModel.getDetails().getFragment());
features.setPath(urlModel.getDetails().getPath());
features.setRedirectChain(urlModel.getDetails().getRedirectChain());
features.setHstsHeader(urlModel.getDetails().getHstsHeader());
features.setSslStripping(urlModel.getDetails().getSslStripping());
features.setHostnameEmbedding(urlModel.getDetails().getHostnameEmbedding());
features.setJavascriptCheck(urlModel.getDetails().getJavascriptCheck());
features.setShorteningService(urlModel.getDetails().getShorteningService());
features.setHasIpAddress(urlModel.getDetails().getHasIpAddress());
features.setTrackingDescriptions(urlModel.getDetails().getTrackingDescriptions());
features.setUrlEncoding(urlModel.getDetails().getUrlEncoding());
features.setHasExecutable(urlModel.getDetails().getHasExecutable());
features.setTls(Math.toIntExact(urlModel.getData().getInfo().getId()));
features.setContents(urlModel.getData().getContents());
return features;
}
// Custom setter for tls (qr_code_type_id) // Custom setter for tls (qr_code_type_id)
public void setTls(Long tls) { public void setTls(Integer tls) {
if (tls != null) { if (tls != null) {
this.tls = tls == 1 ? 0 : tls == 9 ? 1 : tls; this.tls = tls == 1 ? 0.0 : tls == 9 ? 1.0 : tls.doubleValue();
} else { } else {
this.tls = 0L; this.tls = 0.0;
} }
} }
// Custom setter for hostnameEmbedding and other similar columns // Custom setter for hostnameEmbedding and other similar columns
public void setHostnameEmbedding(Long hostnameEmbedding) { public void setHostnameEmbedding(Integer hostnameEmbedding) {
this.hostnameEmbedding = (hostnameEmbedding != null && hostnameEmbedding != 0) ? 1L : 0L; this.hostnameEmbedding = (hostnameEmbedding != null && hostnameEmbedding != 0) ? 1.0 : 0.0;
} }
public void setJavascriptCheck(Long javascriptCheck) { public void setJavascriptCheck(String javascriptCheck) {
this.javascriptCheck = (javascriptCheck != null && javascriptCheck != 0) ? 1L : 0L; this.javascriptCheck = (javascriptCheck != null && !javascriptCheck.isEmpty()) ? 1.0 : 0.0;
} }
public void setShorteningService(Long shorteningService) { public void setShorteningService(String shorteningService) {
this.shorteningService = (shorteningService != null && shorteningService != 0) ? 1L : 0L; this.shorteningService = (shorteningService != null && !shorteningService.isEmpty()) ? 1.0 : 0.0;
} }
public void setHasIpAddress(Long hasIpAddress) { public void setHasIpAddress(String hasIpAddress) {
this.hasIpAddress = (hasIpAddress != null && hasIpAddress != 0) ? 1L : 0L; this.hasIpAddress = (hasIpAddress != null && !hasIpAddress.isEmpty()) ? 1.0 : 0.0;
} }
public void setUrlEncoding(Long urlEncoding) { public void setUrlEncoding(String urlEncoding) {
this.urlEncoding = (urlEncoding != null && urlEncoding != 0) ? 1L : 0L; this.urlEncoding = (urlEncoding != null && !urlEncoding.isEmpty()) ? 1.0 : 0.0;
} }
public void setHasExecutable(Long hasExecutable) { public void setHasExecutable(String hasExecutable) {
this.hasExecutable = (hasExecutable != null && hasExecutable != 0) ? 1L : 0L; this.hasExecutable = (hasExecutable != null && !hasExecutable.isEmpty()) ? 1.0 : 0.0;
} }
public void setTrackingDescriptions(Long trackingDescriptions) { public void setTrackingDescriptions(List<String> trackingDescriptions) {
this.trackingDescriptions = (trackingDescriptions != null && trackingDescriptions != 0) ? 1L : 0L; this.trackingDescriptions = (trackingDescriptions != null && !trackingDescriptions.isEmpty()) ? 1.0 : 0.0;
} }
// Custom setter for sslStripping // Custom setter for sslStripping
public void setSslStripping(String sslStripping) { public void setSslStripping(List<Boolean> sslStripping) {
if (sslStripping != null && "true".equalsIgnoreCase(sslStripping)) { if (sslStripping != null && !sslStripping.isEmpty() && sslStripping.get(0) != null) {
this.sslStripping = 1L; this.sslStripping = sslStripping.get(0) ? 1.0 : 0.0;
} else { } else {
this.sslStripping = 0L; this.sslStripping = 0.0;
} }
} }
// Custom setter for hstsHeader // Custom setter for hstsHeader
public void setHstsHeader(String hstsHeader) { public void setHstsHeader(List<String> hstsHeader) {
if (hstsHeader == null || "0".equals(hstsHeader)) { if (hstsHeader == null || hstsHeader.isEmpty()) {
this.hstsHeader = 0L; this.hstsHeader = 0.0;
} else if (hstsHeader.startsWith("{") && hstsHeader.endsWith("}")) { } else if (hstsHeader.get(0).startsWith("{") && hstsHeader.get(0).endsWith("}")) {
Pattern pattern = Pattern.compile("\"(.*?)\""); Pattern pattern = Pattern.compile("\"(.*?)\"");
Matcher matcher = pattern.matcher(hstsHeader); Matcher matcher = pattern.matcher(hstsHeader.get(0));
if (matcher.find() && matcher.group(1).toLowerCase().contains("no")) { if (matcher.find() && matcher.group(1).toLowerCase().contains("no")) {
this.hstsHeader = 0L; this.hstsHeader = 0.0;
} else { } else {
this.hstsHeader = 1L; this.hstsHeader = 1.0;
} }
} else { } else {
this.hstsHeader = 0L; this.hstsHeader = 1.0;
} }
} }
// Custom setters for calculating string lengths // Custom setters for calculating string lengths
public void setDomain(String domain) { public void setDomain(String domain) {
this.domain = (domain != null) ? (long) domain.length() : 0L; this.domain = (domain != null) ? (double) domain.length() : 0.0;
} }
public void setSubdomain(String subdomain) { public void setSubdomain(String subdomain) {
this.subdomain = (subdomain != null) ? (long) subdomain.length() : 0L; this.subdomain = (subdomain != null) ? (double) subdomain.length() : 0.0;
} }
public void setTopLevelDomain(String topLevelDomain) { public void setTopLevelDomain(String topLevelDomain) {
this.topLevelDomain = (topLevelDomain != null) ? (long) topLevelDomain.length() : 0L; this.topLevelDomain = (topLevelDomain != null) ? (double) topLevelDomain.length() : 0.0;
} }
public void setQuery(String query) { public void setQuery(String query) {
this.query = (query != null) ? (long) query.length() : 0L; this.query = (query != null) ? (double) query.length() : 0.0;
} }
public void setFragment(String fragment) { public void setFragment(String fragment) {
this.fragment = (fragment != null) ? (long) fragment.length() : 0L; this.fragment = (fragment != null) ? (double) fragment.length() : 0.0;
} }
public void setPath(String path) { public void setPath(String path) {
this.path = (path != null) ? (long) path.length() : 0L; this.path = (path != null) ? (double) path.length() : 0.0;
} }
public void setRedirectChain(String redirectChain) { public void setRedirectChain(List<String> redirectChain) {
this.redirectChain = (redirectChain != null) ? (long) redirectChain.length() : 0L; this.redirectChain = (redirectChain != null) ? (double) redirectChain.size() : 0.0;
} }
public void setContents(String contents) { public void setContents(String contents) {
this.contents = (contents != null) ? (long) contents.length() : 0L; this.contents = (contents != null) ? (double) contents.length() : 0.0;
} }
} }

22
src/main/java/com/safeqr/app/spark/service/MLModelService.java Normal file
View File

@@ -0,0 +1,22 @@
package com.safeqr.app.spark.service;
import com.safeqr.app.qrcode.model.URLModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
@Service
public class MLModelService {
private static final Logger logger = LoggerFactory.getLogger(MLModelService.class);
public MLModelService() {
}
public String predict(URLModel urlModel) {
return "haha";
}
}