신은섭(Shin Eun Seop) / Detecting_fraud_clicks

Go to a project
Toggle navigation Toggle navigation pinning
신은섭(Shin Eun Seop)
ffc8ae86 1 parent 7a23214c

att1

Inline Side-by-side
Showing 6 changed files with 252 additions and 4 deletions
1 <?xml version="1.0" encoding="UTF-8"?> 1 <?xml version="1.0" encoding="UTF-8"?>
2 -<module type="JAVA_MODULE" version="4"> 2 +<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
3 - <component name="NewModuleRootManager" inherit-compiler-output="true"> 3 + <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
4 - <exclude-output /> 4 + <output url="file://$MODULE_DIR$/target/classes" />
5 - <content url="file://$MODULE_DIR$" /> 5 + <output-test url="file://$MODULE_DIR$/target/test-classes" />
6 + <content url="file://$MODULE_DIR$">
7 + <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
8 + <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
9 + <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
10 + <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
11 + <excludeFolder url="file://$MODULE_DIR$/target" />
12 + </content>
6 <orderEntry type="inheritedJdk" /> 13 <orderEntry type="inheritedJdk" />
7 <orderEntry type="sourceFolder" forTests="false" /> 14 <orderEntry type="sourceFolder" forTests="false" />
15 + <orderEntry type="library" name="Maven: org.apache.spark:spark-core_2.11:2.3.0" level="project" />
16 + <orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.7" level="project" />
17 + <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.13" level="project" />
18 + <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.13" level="project" />
19 + <orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />
20 + <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
21 + <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
22 + <orderEntry type="library" name="Maven: org.apache.avro:avro-mapred:hadoop2:1.7.7" level="project" />
23 + <orderEntry type="library" name="Maven: org.apache.avro:avro-ipc:1.7.7" level="project" />
24 + <orderEntry type="library" name="Maven: org.apache.avro:avro-ipc:tests:1.7.7" level="project" />
25 + <orderEntry type="library" name="Maven: com.twitter:chill_2.11:0.8.4" level="project" />
26 + <orderEntry type="library" name="Maven: com.esotericsoftware:kryo-shaded:3.0.3" level="project" />
27 + <orderEntry type="library" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" />
28 + <orderEntry type="library" name="Maven: org.objenesis:objenesis:2.1" level="project" />
29 + <orderEntry type="library" name="Maven: com.twitter:chill-java:0.8.4" level="project" />
30 + <orderEntry type="library" name="Maven: org.apache.xbean:xbean-asm5-shaded:4.4" level="project" />
31 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.6.5" level="project" />
32 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.6.5" level="project" />
33 + <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
34 + <orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
35 + <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
36 + <orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
37 + <orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
38 + <orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
39 + <orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
40 + <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
41 + <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
42 + <orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.4" level="project" />
43 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.6.5" level="project" />
44 + <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
45 + <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
46 + <orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
47 + <orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
48 + <orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.6.0" level="project" />
49 + <orderEntry type="library" name="Maven: org.htrace:htrace-core:3.0.4" level="project" />
50 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.5" level="project" />
51 + <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26" level="project" />
52 + <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
53 + <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
54 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.5" level="project" />
55 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.5" level="project" />
56 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.5" level="project" />
57 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.5" level="project" />
58 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.5" level="project" />
59 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.5" level="project" />
60 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.5" level="project" />
61 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.5" level="project" />
62 + <orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
63 + <orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
64 + <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.13" level="project" />
65 + <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.9.13" level="project" />
66 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.5" level="project" />
67 + <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.5" level="project" />
68 + <orderEntry type="library" name="Maven: org.apache.spark:spark-launcher_2.11:2.3.0" level="project" />
69 + <orderEntry type="library" name="Maven: org.apache.spark:spark-kvstore_2.11:2.3.0" level="project" />
70 + <orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
71 + <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.6.7" level="project" />
72 + <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.6.7" level="project" />
73 + <orderEntry type="library" name="Maven: org.apache.spark:spark-network-common_2.11:2.3.0" level="project" />
74 + <orderEntry type="library" name="Maven: org.apache.spark:spark-network-shuffle_2.11:2.3.0" level="project" />
75 + <orderEntry type="library" name="Maven: org.apache.spark:spark-unsafe_2.11:2.3.0" level="project" />
76 + <orderEntry type="library" name="Maven: net.java.dev.jets3t:jets3t:0.9.4" level="project" />
77 + <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.1" level="project" />
78 + <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5" level="project" />
79 + <orderEntry type="library" name="Maven: commons-codec:commons-codec:2.0-SNAPSHOT" level="project" />
80 + <orderEntry type="library" name="Maven: javax.activation:activation:1.1.1" level="project" />
81 + <orderEntry type="library" name="Maven: org.bouncycastle:bcprov-jdk15on:1.52" level="project" />
82 + <orderEntry type="library" name="Maven: com.jamesmurty.utils:java-xmlbuilder:1.1" level="project" />
83 + <orderEntry type="library" name="Maven: net.iharder:base64:2.3.8" level="project" />
84 + <orderEntry type="library" name="Maven: org.apache.curator:curator-recipes:2.6.0" level="project" />
85 + <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.6.0" level="project" />
86 + <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
87 + <orderEntry type="library" name="Maven: com.google.guava:guava:16.0.1" level="project" />
88 + <orderEntry type="library" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
89 + <orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.5" level="project" />
90 + <orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.4.1" level="project" />
91 + <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
92 + <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.16" level="project" />
93 + <orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.16" level="project" />
94 + <orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.16" level="project" />
95 + <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
96 + <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.16" level="project" />
97 + <orderEntry type="library" name="Maven: com.ning:compress-lzf:1.0.3" level="project" />
98 + <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.1.2.6" level="project" />
99 + <orderEntry type="library" name="Maven: org.lz4:lz4-java:1.4.0" level="project" />
100 + <orderEntry type="library" name="Maven: com.github.luben:zstd-jni:1.3.2-2" level="project" />
101 + <orderEntry type="library" name="Maven: org.roaringbitmap:RoaringBitmap:0.5.11" level="project" />
102 + <orderEntry type="library" name="Maven: commons-net:commons-net:2.2" level="project" />
103 + <orderEntry type="library" name="Maven: org.scala-lang:scala-library:2.11.8" level="project" />
104 + <orderEntry type="library" name="Maven: org.json4s:json4s-jackson_2.11:3.2.11" level="project" />
105 + <orderEntry type="library" name="Maven: org.json4s:json4s-core_2.11:3.2.11" level="project" />
106 + <orderEntry type="library" name="Maven: org.json4s:json4s-ast_2.11:3.2.11" level="project" />
107 + <orderEntry type="library" name="Maven: org.scala-lang:scalap:2.11.0" level="project" />
108 + <orderEntry type="library" name="Maven: org.scala-lang:scala-compiler:2.11.0" level="project" />
109 + <orderEntry type="library" name="Maven: org.scala-lang.modules:scala-xml_2.11:1.0.1" level="project" />
110 + <orderEntry type="library" name="Maven: org.glassfish.jersey.core:jersey-client:2.22.2" level="project" />
111 + <orderEntry type="library" name="Maven: javax.ws.rs:javax.ws.rs-api:2.0.1" level="project" />
112 + <orderEntry type="library" name="Maven: org.glassfish.hk2:hk2-api:2.4.0-b34" level="project" />
113 + <orderEntry type="library" name="Maven: org.glassfish.hk2:hk2-utils:2.4.0-b34" level="project" />
114 + <orderEntry type="library" name="Maven: org.glassfish.hk2.external:aopalliance-repackaged:2.4.0-b34" level="project" />
115 + <orderEntry type="library" name="Maven: org.glassfish.hk2.external:javax.inject:2.4.0-b34" level="project" />
116 + <orderEntry type="library" name="Maven: org.glassfish.hk2:hk2-locator:2.4.0-b34" level="project" />
117 + <orderEntry type="library" name="Maven: org.javassist:javassist:3.18.1-GA" level="project" />
118 + <orderEntry type="library" name="Maven: org.glassfish.jersey.core:jersey-common:2.22.2" level="project" />
119 + <orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.2" level="project" />
120 + <orderEntry type="library" name="Maven: org.glassfish.jersey.bundles.repackaged:jersey-guava:2.22.2" level="project" />
121 + <orderEntry type="library" name="Maven: org.glassfish.hk2:osgi-resource-locator:1.0.1" level="project" />
122 + <orderEntry type="library" name="Maven: org.glassfish.jersey.core:jersey-server:2.22.2" level="project" />
123 + <orderEntry type="library" name="Maven: org.glassfish.jersey.media:jersey-media-jaxb:2.22.2" level="project" />
124 + <orderEntry type="library" name="Maven: javax.validation:validation-api:1.1.0.Final" level="project" />
125 + <orderEntry type="library" name="Maven: org.glassfish.jersey.containers:jersey-container-servlet:2.22.2" level="project" />
126 + <orderEntry type="library" name="Maven: org.glassfish.jersey.containers:jersey-container-servlet-core:2.22.2" level="project" />
127 + <orderEntry type="library" name="Maven: io.netty:netty-all:4.1.17.Final" level="project" />
128 + <orderEntry type="library" name="Maven: io.netty:netty:3.9.9.Final" level="project" />
129 + <orderEntry type="library" name="Maven: com.clearspring.analytics:stream:2.7.0" level="project" />
130 + <orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-core:3.1.5" level="project" />
131 + <orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-jvm:3.1.5" level="project" />
132 + <orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-json:3.1.5" level="project" />
133 + <orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-graphite:3.1.5" level="project" />
134 + <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.6.7.1" level="project" />
135 + <orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-scala_2.11:2.6.7.1" level="project" />
136 + <orderEntry type="library" name="Maven: org.scala-lang:scala-reflect:2.11.8" level="project" />
137 + <orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-paranamer:2.7.9" level="project" />
138 + <orderEntry type="library" name="Maven: org.apache.ivy:ivy:2.4.0" level="project" />
139 + <orderEntry type="library" name="Maven: oro:oro:2.0.8" level="project" />
140 + <orderEntry type="library" name="Maven: net.razorvine:pyrolite:4.13" level="project" />
141 + <orderEntry type="library" name="Maven: net.sf.py4j:py4j:0.10.6" level="project" />
142 + <orderEntry type="library" name="Maven: org.apache.spark:spark-tags_2.11:2.3.0" level="project" />
143 + <orderEntry type="library" name="Maven: org.apache.commons:commons-crypto:1.0.0" level="project" />
144 + <orderEntry type="library" name="Maven: org.spark-project.spark:unused:1.0.0" level="project" />
145 + <orderEntry type="library" name="Maven: org.apache.spark:spark-sql_2.11:2.3.0" level="project" />
146 + <orderEntry type="library" name="Maven: com.univocity:univocity-parsers:2.5.9" level="project" />
147 + <orderEntry type="library" name="Maven: org.apache.spark:spark-sketch_2.11:2.3.0" level="project" />
148 + <orderEntry type="library" name="Maven: org.apache.spark:spark-catalyst_2.11:2.3.0" level="project" />
149 + <orderEntry type="library" name="Maven: org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4" level="project" />
150 + <orderEntry type="library" name="Maven: org.codehaus.janino:janino:3.0.8" level="project" />
151 + <orderEntry type="library" name="Maven: org.codehaus.janino:commons-compiler:3.0.8" level="project" />
152 + <orderEntry type="library" name="Maven: org.antlr:antlr4-runtime:4.7" level="project" />
153 + <orderEntry type="library" name="Maven: org.apache.orc:orc-core:nohive:1.4.1" level="project" />
154 + <orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
155 + <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
156 + <orderEntry type="library" name="Maven: io.airlift:aircompressor:0.8" level="project" />
157 + <orderEntry type="library" name="Maven: org.apache.orc:orc-mapreduce:nohive:1.4.1" level="project" />
158 + <orderEntry type="library" name="Maven: org.apache.parquet:parquet-column:1.8.2" level="project" />
159 + <orderEntry type="library" name="Maven: org.apache.parquet:parquet-common:1.8.2" level="project" />
160 + <orderEntry type="library" name="Maven: org.apache.parquet:parquet-encoding:1.8.2" level="project" />
161 + <orderEntry type="library" name="Maven: org.apache.parquet:parquet-hadoop:1.8.2" level="project" />
162 + <orderEntry type="library" name="Maven: org.apache.parquet:parquet-format:2.3.1" level="project" />
163 + <orderEntry type="library" name="Maven: org.apache.parquet:parquet-jackson:1.8.2" level="project" />
164 + <orderEntry type="library" name="Maven: org.apache.arrow:arrow-vector:0.8.0" level="project" />
165 + <orderEntry type="library" name="Maven: org.apache.arrow:arrow-format:0.8.0" level="project" />
166 + <orderEntry type="library" name="Maven: org.apache.arrow:arrow-memory:0.8.0" level="project" />
167 + <orderEntry type="library" name="Maven: joda-time:joda-time:2.9.9" level="project" />
168 + <orderEntry type="library" name="Maven: com.carrotsearch:hppc:0.7.2" level="project" />
169 + <orderEntry type="library" name="Maven: com.vlkan:flatbuffers:1.2.0-3f79e055" level="project" />
170 + <orderEntry type="library" name="Maven: com.databricks:spark-csv_2.11:1.5.0" level="project" />
171 + <orderEntry type="library" name="Maven: org.apache.commons:commons-csv:1.1" level="project" />
8 </component> 172 </component>
9 </module> 173 </module>
...\ No newline at end of file ...\ No newline at end of file
......
1 +<?xml version="1.0" encoding="UTF-8"?>
2 +<project version="4">
3 + <component name="CompilerConfiguration">
4 + <annotationProcessing>
5 + <profile name="Maven default annotation processors profile" enabled="true">
6 + <sourceOutputDir name="target/generated-sources/annotations" />
7 + <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
8 + <outputRelativeToContentRoot value="true" />
9 + <module name="Detecting_fraud_clicks" />
10 + </profile>
11 + </annotationProcessing>
12 + <bytecodeTargetLevel target="9">
13 + <module name="Detecting_fraud_clicks" target="1.8" />
14 + </bytecodeTargetLevel>
15 + </component>
16 +</project>
...\ No newline at end of file ...\ No newline at end of file
...@@ -3,4 +3,14 @@ ...@@ -3,4 +3,14 @@
3 <component name="JavaScriptSettings"> 3 <component name="JavaScriptSettings">
4 <option name="languageLevel" value="ES6" /> 4 <option name="languageLevel" value="ES6" />
5 </component> 5 </component>
6 + <component name="MavenProjectsManager">
7 + <option name="originalFiles">
8 + <list>
9 + <option value="$PROJECT_DIR$/pom.xml" />
10 + </list>
11 + </option>
12 + </component>
13 + <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="false" project-jdk-name="1.8" project-jdk-type="JavaSDK">
14 + <output url="file:///tmp" />
15 + </component>
6 </project> 16 </project>
...\ No newline at end of file ...\ No newline at end of file
......
...@@ -17,6 +17,33 @@ ...@@ -17,6 +17,33 @@
17 <version>2.3.0</version> 17 <version>2.3.0</version>
18 </dependency> 18 </dependency>
19 19
20 + <dependency>
21 + <groupId>org.apache.spark</groupId>
22 + <artifactId>spark-sql_2.11</artifactId>
23 + <version>2.3.0</version>
24 + </dependency>
25 +
26 + <dependency>
27 + <groupId>com.databricks</groupId>
28 + <artifactId>spark-csv_2.11</artifactId>
29 + <version>1.5.0</version>
30 + </dependency>
31 +
20 </dependencies> 32 </dependencies>
21 33
34 + <build>
35 + <plugins>
36 + <plugin>
37 + <groupId>org.apache.maven.plugins</groupId>
38 + <artifactId>maven-compiler-plugin</artifactId>
39 + <version>3.6.1</version>
40 + <configuration>
41 + <source>1.8</source>
42 + <target>1.8</target>
43 + </configuration>
44 + </plugin>
45 + </plugins>
46 + </build>
47 +
48 +
22 </project> 49 </project>
...\ No newline at end of file ...\ No newline at end of file
......
1 +import org.apache.spark.SparkConf;
2 +import org.apache.spark.api.java.JavaSparkContext;
3 +import org.apache.spark.sql.*;
4 +import org.apache.spark.sql.Row;
5 +
6 +
7 +public class AvgAdvTime {
8 +
9 + public static void main(String[] args) throws Exception {
10 +
11 + SparkSession spark = SparkSession
12 + .builder()
13 + .appName("Java Spark SQL basic example")
14 + .getOrCreate();
15 +
16 + Dataset<Row> df = spark.read().format("csv")
17 + .option("inferSchema", "true")
18 + .option("header", "true")
19 + .load("train_sample.csv");
20 +
21 + df.show();
22 + df.createOrReplaceTempView("logs");
23 +
24 + Dataset<Row> ds = spark.sql("SELECT ip, app, click_time, is_attributed" +
25 + "FROM logs " +
26 + "ORDER BY click_time");
27 + ds.show();
28 +
29 + System.out.println();
30 + }
31 +}
...\ No newline at end of file ...\ No newline at end of file
This diff could not be displayed because it is too large.