Skip to content

Commit f6532fd

Browse files
author
Jony Santos
committed
initial commit
1 parent 570779d commit f6532fd

File tree

12 files changed

+116888
-0
lines changed

12 files changed

+116888
-0
lines changed

Diff for: ‎access.log

+116,484
Large diffs are not rendered by default.

Diff for: ‎lombok.config

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
lombok.log.fieldName=LOGGER

Diff for: ‎pom.xml

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
6+
<groupId>br.com.jonyfs</groupId>
7+
<artifactId>parser</artifactId>
8+
<version>0.0.1-SNAPSHOT</version>
9+
<packaging>jar</packaging>
10+
11+
<name>parser</name>
12+
<description>Java Parser Example</description>
13+
14+
<parent>
15+
<groupId>org.springframework.boot</groupId>
16+
<artifactId>spring-boot-starter-parent</artifactId>
17+
<version>2.0.0.RC1</version>
18+
<relativePath/> <!-- lookup parent from repository -->
19+
</parent>
20+
21+
<properties>
22+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
23+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
24+
<java.version>1.8</java.version>
25+
</properties>
26+
27+
<dependencies>
28+
<dependency>
29+
<groupId>org.springframework.shell</groupId>
30+
<artifactId>spring-shell-starter</artifactId>
31+
<version>2.0.0.M2</version>
32+
</dependency>
33+
<dependency>
34+
<groupId>org.springframework.boot</groupId>
35+
<artifactId>spring-boot-starter-data-jpa</artifactId>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.hsqldb</groupId>
39+
<artifactId>hsqldb</artifactId>
40+
<scope>runtime</scope>
41+
</dependency>
42+
<dependency>
43+
<groupId>org.projectlombok</groupId>
44+
<artifactId>lombok</artifactId>
45+
<optional>true</optional>
46+
</dependency>
47+
<dependency>
48+
<groupId>org.springframework.boot</groupId>
49+
<artifactId>spring-boot-starter-test</artifactId>
50+
<scope>test</scope>
51+
</dependency>
52+
<dependency>
53+
<groupId>com.querydsl</groupId>
54+
<artifactId>querydsl-apt</artifactId>
55+
<version>4.1.4</version>
56+
</dependency>
57+
<dependency>
58+
<groupId>com.querydsl</groupId>
59+
<artifactId>querydsl-jpa</artifactId>
60+
<version>4.1.4</version>
61+
</dependency>
62+
</dependencies>
63+
64+
<build>
65+
<plugins>
66+
<plugin>
67+
<groupId>org.springframework.boot</groupId>
68+
<artifactId>spring-boot-maven-plugin</artifactId>
69+
</plugin>
70+
<plugin>
71+
<groupId>com.mysema.maven</groupId>
72+
<artifactId>apt-maven-plugin</artifactId>
73+
<version>1.1.3</version>
74+
<executions>
75+
<execution>
76+
<goals>
77+
<goal>process</goal>
78+
</goals>
79+
<configuration>
80+
<outputDirectory>target/generated-sources/java</outputDirectory>
81+
<processor>com.querydsl.apt.QuerydslAnnotationProcessor</processor>
82+
</configuration>
83+
</execution>
84+
</executions>
85+
</plugin>
86+
<plugin>
87+
<groupId>org.codehaus.mojo</groupId>
88+
<artifactId>build-helper-maven-plugin</artifactId>
89+
<executions>
90+
<execution>
91+
<id>add-source</id>
92+
<phase>generate-sources</phase>
93+
<goals>
94+
<goal>add-source</goal>
95+
</goals>
96+
<configuration>
97+
<sources>
98+
<source>${project.build.directory}/generated-sources/java</source>
99+
</sources>
100+
</configuration>
101+
</execution>
102+
</executions>
103+
</plugin>
104+
</plugins>
105+
</build>
106+
107+
<repositories>
108+
<repository>
109+
<id>spring-milestones</id>
110+
<name>Spring Milestones</name>
111+
<url>https://repo.spring.io/milestone</url>
112+
<snapshots>
113+
<enabled>false</enabled>
114+
</snapshots>
115+
</repository>
116+
<repository>
117+
<id>spring-snapshots</id>
118+
<name>Spring Snapshots</name>
119+
<url>https://repo.spring.io/snapshot</url>
120+
<snapshots>
121+
<enabled>true</enabled>
122+
</snapshots>
123+
</repository>
124+
</repositories>
125+
126+
<pluginRepositories>
127+
<pluginRepository>
128+
<id>spring-snapshots</id>
129+
<name>Spring Snapshots</name>
130+
<url>https://repo.spring.io/snapshot</url>
131+
<snapshots>
132+
<enabled>true</enabled>
133+
</snapshots>
134+
</pluginRepository>
135+
<pluginRepository>
136+
<id>spring-milestones</id>
137+
<name>Spring Milestones</name>
138+
<url>https://repo.spring.io/milestone</url>
139+
<snapshots>
140+
<enabled>false</enabled>
141+
</snapshots>
142+
</pluginRepository>
143+
</pluginRepositories>
144+
145+
146+
</project>

Diff for: ‎src/main/java/br/com/jonyfs/parser/Duration.java

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package br.com.jonyfs.parser;
2+
3+
public enum Duration {
4+
hourly, daily
5+
}

Diff for: ‎src/main/java/br/com/jonyfs/parser/ParserApplication.java

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package br.com.jonyfs.parser;
2+
3+
import org.springframework.boot.SpringApplication;
4+
import org.springframework.boot.autoconfigure.SpringBootApplication;
5+
6+
@SpringBootApplication
7+
public class ParserApplication {
8+
9+
public static void main(String[] args) {
10+
SpringApplication.run(ParserApplication.class, args);
11+
}
12+
}

Diff for: ‎src/main/java/br/com/jonyfs/parser/ParserCommands.java

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package br.com.jonyfs.parser;
2+
3+
import br.com.jonyfs.parser.log.LogRepository;
4+
import br.com.jonyfs.parser.log.LogService;
5+
import lombok.extern.slf4j.Slf4j;
6+
import org.springframework.shell.standard.ShellComponent;
7+
import org.springframework.shell.standard.ShellMethod;
8+
import org.springframework.shell.standard.ShellOption;
9+
10+
import javax.annotation.Resource;
11+
import java.util.List;
12+
13+
@Slf4j
14+
@ShellComponent
15+
public class ParserCommands {
16+
17+
@Resource
18+
LogService logService;
19+
20+
@Resource
21+
LogRepository logRepository;
22+
23+
@ShellMethod("Parse the log file and search data. Example: parse path-to-log-file start-date(in yyyy-MM-dd.HH.mm.ss format) duration(hourly or daily) threshold(greather than 0)")
24+
public List parse(
25+
@ShellOption() String accesslog,
26+
@ShellOption() String startDate,
27+
@ShellOption() Duration duration,
28+
@ShellOption() Long threshold
29+
) {
30+
load(accesslog);
31+
32+
return query(startDate, duration, threshold);
33+
}
34+
35+
@ShellMethod("Load data file. Example: load path-to-log-file")
36+
public String load(
37+
@ShellOption() String accesslog
38+
) {
39+
logService.load(accesslog);
40+
return "Loaded";
41+
}
42+
43+
@ShellMethod("count data rows on database")
44+
public Long count(
45+
) {
46+
return logRepository.count();
47+
}
48+
49+
@ShellMethod("clear all data rows on database")
50+
public String clean(
51+
) {
52+
logRepository.deleteAll();
53+
return "OK!";
54+
}
55+
56+
@ShellMethod("Search data from database. Example: query start-date(in yyyy-MM-dd.HH.mm.ss format) duration(hourly or daily) threshold(greather than 0)")
57+
public List query(
58+
@ShellOption() String startDate,
59+
@ShellOption() Duration duration,
60+
@ShellOption() Long threshold
61+
) {
62+
LOGGER.info("Searching with startDate: {}, duration: {} and threshold: {}...", startDate, duration, threshold);
63+
return logService.search(startDate, duration, threshold);
64+
}
65+
}

Diff for: ‎src/main/java/br/com/jonyfs/parser/ParserConstants.java

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package br.com.jonyfs.parser;
2+
3+
import java.time.format.DateTimeFormatter;
4+
5+
public class ParserConstants {
6+
public static final DateTimeFormatter FORMATTER_LOG = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
7+
8+
public static final DateTimeFormatter FORMATTER_QUERY = DateTimeFormatter.ofPattern("yyyy-MM-dd.HH:mm:ss");
9+
}

Diff for: ‎src/main/java/br/com/jonyfs/parser/log/Log.java

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package br.com.jonyfs.parser.log;
2+
3+
import com.querydsl.core.annotations.QueryEntity;
4+
import lombok.Setter;
5+
import lombok.ToString;
6+
import org.springframework.data.domain.Persistable;
7+
import org.springframework.lang.Nullable;
8+
9+
import javax.persistence.*;
10+
import javax.validation.constraints.Min;
11+
import javax.validation.constraints.NotEmpty;
12+
import javax.validation.constraints.NotNull;
13+
import java.time.LocalDateTime;
14+
15+
@QueryEntity
16+
@ToString
17+
@Entity
18+
public class Log implements Persistable<Long> {
19+
20+
@Id
21+
@GeneratedValue(strategy = GenerationType.AUTO)
22+
@Column
23+
private Long id;
24+
25+
@Column
26+
@Setter
27+
private LocalDateTime date;
28+
29+
@Column
30+
@Setter
31+
@NotEmpty
32+
private String ip;
33+
34+
@Column
35+
@Setter
36+
@NotEmpty
37+
private String request;
38+
39+
@Column
40+
@Setter
41+
@NotNull
42+
@Min(0)
43+
private Long status;
44+
45+
@Column
46+
@Setter
47+
@NotEmpty
48+
private String userAgent;
49+
50+
@Nullable
51+
@Override
52+
public Long getId() {
53+
return null;
54+
}
55+
56+
@Override
57+
public boolean isNew() {
58+
return id == null;
59+
}
60+
}

Diff for: ‎src/main/java/br/com/jonyfs/parser/log/LogRepository.java

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package br.com.jonyfs.parser.log;
2+
3+
import org.springframework.data.querydsl.QuerydslPredicateExecutor;
4+
import org.springframework.data.querydsl.binding.QuerydslBinderCustomizer;
5+
import org.springframework.data.querydsl.binding.QuerydslBindings;
6+
import org.springframework.data.repository.CrudRepository;
7+
import org.springframework.stereotype.Repository;
8+
9+
@Repository
10+
public interface LogRepository extends CrudRepository<Log, Long>, QuerydslPredicateExecutor<Log>, QuerydslBinderCustomizer<QLog> {
11+
@Override
12+
default void customize(QuerydslBindings bindings, QLog root) {
13+
14+
}
15+
}

Diff for: ‎src/main/java/br/com/jonyfs/parser/log/LogService.java

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package br.com.jonyfs.parser.log;
2+
3+
import br.com.jonyfs.parser.Duration;
4+
import br.com.jonyfs.parser.ParserConstants;
5+
import com.querydsl.jpa.impl.JPAQuery;
6+
import com.querydsl.jpa.impl.JPAQueryFactory;
7+
import lombok.extern.slf4j.Slf4j;
8+
import org.springframework.stereotype.Component;
9+
10+
import javax.annotation.Resource;
11+
import javax.persistence.EntityManagerFactory;
12+
import java.io.IOException;
13+
import java.nio.file.Files;
14+
import java.nio.file.Paths;
15+
import java.time.LocalDateTime;
16+
import java.util.List;
17+
import java.util.stream.Stream;
18+
19+
@Slf4j
20+
@Component
21+
public class LogService {
22+
23+
@Resource
24+
LogRepository logRepository;
25+
26+
@Resource
27+
EntityManagerFactory entityManagerFactory;
28+
29+
public void load(String accesslog) {
30+
LOGGER.info("Reading file {}...", accesslog);
31+
try (Stream<String> stream = Files.lines(Paths.get(accesslog))) {
32+
33+
stream.forEach(this::parse);
34+
35+
} catch (IOException e) {
36+
throw new RuntimeException("Fail to open file ", e);
37+
}
38+
LOGGER.info("OK!");
39+
}
40+
41+
private void parse(String line) {
42+
int i = 0;
43+
String[] data = line.split("\\|");
44+
Log log = new Log();
45+
log.setDate(LocalDateTime.parse(data[i++], ParserConstants.FORMATTER_LOG));
46+
log.setIp(data[i++]);
47+
log.setRequest(data[i++]);
48+
log.setStatus(Long.valueOf(data[i++]));
49+
log.setUserAgent(data[i++]);
50+
logRepository.save(log);
51+
}
52+
53+
public List search(String startDate,
54+
Duration duration,
55+
Long threshold) {
56+
QLog log = QLog.log;
57+
LocalDateTime initialDate;
58+
LocalDateTime finalDate;
59+
initialDate = LocalDateTime.parse(startDate, ParserConstants.FORMATTER_QUERY);
60+
if (Duration.hourly.equals(duration)) {
61+
finalDate = initialDate.plusHours(1);
62+
} else {
63+
finalDate = initialDate.plusDays(1);
64+
}
65+
66+
JPAQuery query = new JPAQueryFactory(entityManagerFactory.createEntityManager())
67+
.select(log.ip, log.count())
68+
.from(log)
69+
.where(log.date.between(initialDate, finalDate))
70+
.groupBy(log.ip)
71+
.having(log.count().goe(threshold))
72+
.orderBy(log.count().desc());
73+
return query.fetch();
74+
}
75+
}

Diff for: ‎src/main/resources/application.properties

Whitespace-only changes.

0 commit comments

Comments
 (0)