Skip to content

Commit 2c6ac37

Browse files
author
Anish Chakraborty
committed
Initial Commit
added code files
0 parents  commit 2c6ac37

File tree

13 files changed

+1216
-0
lines changed

13 files changed

+1216
-0
lines changed

‎.gitignore

+199
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
2+
# Created by https://www.gitignore.io/api/scala,osx,windows,eclipse,intellij,maven
3+
4+
### Eclipse ###
5+
6+
.metadata
7+
bin/
8+
tmp/
9+
*.tmp
10+
*.bak
11+
*.swp
12+
*~.nib
13+
local.properties
14+
.settings/
15+
.loadpath
16+
.recommenders
17+
18+
# Eclipse Core
19+
.project
20+
21+
# External tool builders
22+
.externalToolBuilders/
23+
24+
# Locally stored "Eclipse launch configurations"
25+
*.launch
26+
27+
# PyDev specific (Python IDE for Eclipse)
28+
*.pydevproject
29+
30+
# CDT-specific (C/C++ Development Tooling)
31+
.cproject
32+
33+
# JDT-specific (Eclipse Java Development Tools)
34+
.classpath
35+
36+
# Java annotation processor (APT)
37+
.factorypath
38+
39+
# PDT-specific (PHP Development Tools)
40+
.buildpath
41+
42+
# sbteclipse plugin
43+
.target
44+
45+
# Tern plugin
46+
.tern-project
47+
48+
# TeXlipse plugin
49+
.texlipse
50+
51+
# STS (Spring Tool Suite)
52+
.springBeans
53+
54+
# Code Recommenders
55+
.recommenders/
56+
57+
### Intellij ###
58+
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
59+
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
60+
61+
# User-specific stuff:
62+
.idea/**/workspace.xml
63+
.idea/**/tasks.xml
64+
65+
# Sensitive or high-churn files:
66+
.idea/**/dataSources/
67+
.idea/**/dataSources.ids
68+
.idea/**/dataSources.xml
69+
.idea/**/dataSources.local.xml
70+
.idea/**/sqlDataSources.xml
71+
.idea/**/dynamic.xml
72+
.idea/**/uiDesigner.xml
73+
74+
# Gradle:
75+
.idea/**/gradle.xml
76+
.idea/**/libraries
77+
78+
# Mongo Explorer plugin:
79+
.idea/**/mongoSettings.xml
80+
81+
## File-based project format:
82+
*.iws
83+
84+
## Plugin-specific files:
85+
86+
# IntelliJ
87+
/out/
88+
89+
# mpeltonen/sbt-idea plugin
90+
.idea_modules/
91+
92+
# JIRA plugin
93+
atlassian-ide-plugin.xml
94+
95+
# Crashlytics plugin (for Android Studio and IntelliJ)
96+
com_crashlytics_export_strings.xml
97+
crashlytics.properties
98+
crashlytics-build.properties
99+
fabric.properties
100+
101+
### Intellij Patch ###
102+
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
103+
104+
# *.iml
105+
# modules.xml
106+
# .idea/misc.xml
107+
# *.ipr
108+
109+
### Maven ###
110+
target/
111+
pom.xml.tag
112+
pom.xml.releaseBackup
113+
pom.xml.versionsBackup
114+
pom.xml.next
115+
release.properties
116+
dependency-reduced-pom.xml
117+
buildNumber.properties
118+
.mvn/timing.properties
119+
120+
# Exclude maven wrapper
121+
!/.mvn/wrapper/maven-wrapper.jar
122+
123+
### OSX ###
124+
*.DS_Store
125+
.AppleDouble
126+
.LSOverride
127+
128+
# Icon must end with two \r
129+
Icon
130+
131+
132+
# Thumbnails
133+
._*
134+
135+
# Files that might appear in the root of a volume
136+
.DocumentRevisions-V100
137+
.fseventsd
138+
.Spotlight-V100
139+
.TemporaryItems
140+
.Trashes
141+
.VolumeIcon.icns
142+
.com.apple.timemachine.donotpresent
143+
144+
# Directories potentially created on remote AFP share
145+
.AppleDB
146+
.AppleDesktop
147+
Network Trash Folder
148+
Temporary Items
149+
.apdisk
150+
151+
### Scala ###
152+
*.class
153+
*.log
154+
155+
# sbt specific
156+
.cache
157+
.history
158+
.lib/
159+
dist/*
160+
lib_managed/
161+
src_managed/
162+
project/boot/
163+
project/plugins/project/
164+
165+
# Scala-IDE specific
166+
.ensime
167+
.ensime_cache/
168+
.scala_dependencies
169+
.worksheet
170+
171+
# ENSIME specific
172+
173+
### Windows ###
174+
# Windows thumbnail cache files
175+
Thumbs.db
176+
ehthumbs.db
177+
ehthumbs_vista.db
178+
179+
# Folder config file
180+
Desktop.ini
181+
182+
# Recycle Bin used on file shares
183+
$RECYCLE.BIN/
184+
185+
# Windows Installer files
186+
*.cab
187+
*.msi
188+
*.msm
189+
*.msp
190+
191+
# Windows shortcuts
192+
*.lnk
193+
194+
# End of https://www.gitignore.io/api/scala,osx,windows,eclipse,intellij,maven
195+
196+
# Project related files
197+
.idea/*
198+
*.iml
199+
spark-warehouse/*

‎pom.xml

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<groupId>org.anish.spark</groupId>
5+
<artifactId>spark2-etl-examples</artifactId>
6+
<version>1.0-SNAPSHOT</version>
7+
<inceptionYear>2008</inceptionYear>
8+
<properties>
9+
<spark.version>2.0.0</spark.version>
10+
<scala.tools.version>2.11</scala.tools.version>
11+
<scala.version>2.11.6</scala.version>
12+
</properties>
13+
14+
<repositories>
15+
<repository>
16+
<id>scala-tools.org</id>
17+
<name>Scala-Tools Maven2 Repository</name>
18+
<url>http://scala-tools.org/repo-releases</url>
19+
</repository>
20+
</repositories>
21+
22+
<pluginRepositories>
23+
<pluginRepository>
24+
<id>scala-tools.org</id>
25+
<name>Scala-Tools Maven2 Repository</name>
26+
<url>http://scala-tools.org/repo-releases</url>
27+
</pluginRepository>
28+
</pluginRepositories>
29+
30+
<dependencies>
31+
<dependency>
32+
<groupId>org.apache.spark</groupId>
33+
<artifactId>spark-core_${scala.tools.version}</artifactId>
34+
<version>${spark.version}</version>
35+
</dependency>
36+
37+
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.11 -->
38+
<dependency>
39+
<groupId>org.apache.spark</groupId>
40+
<artifactId>spark-sql_${scala.tools.version}</artifactId>
41+
<version>${spark.version}</version>
42+
</dependency>
43+
44+
<dependency>
45+
<groupId>com.databricks</groupId>
46+
<artifactId>spark-avro_${scala.tools.version}</artifactId>
47+
<version>3.1.0</version>
48+
</dependency>
49+
50+
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-hive_2.10 -->
51+
<dependency>
52+
<groupId>org.apache.spark</groupId>
53+
<artifactId>spark-hive_${scala.tools.version}</artifactId>
54+
<version>${spark.version}</version>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.scala-lang</groupId>
58+
<artifactId>scala-library</artifactId>
59+
<version>${scala.version}</version>
60+
</dependency>
61+
<dependency>
62+
<groupId>junit</groupId>
63+
<artifactId>junit</artifactId>
64+
<version>4.4</version>
65+
<scope>test</scope>
66+
</dependency>
67+
<dependency>
68+
<groupId>org.scala-tools.testing</groupId>
69+
<artifactId>specs</artifactId>
70+
<version>1.6.2.2_1.5.0</version>
71+
<scope>test</scope>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.scalatest</groupId>
75+
<artifactId>scalatest_${scala.tools.version}</artifactId>
76+
<version>2.2.5</version>
77+
</dependency>
78+
</dependencies>
79+
80+
<build>
81+
<sourceDirectory>src/main/scala</sourceDirectory>
82+
<testSourceDirectory>src/test/scala</testSourceDirectory>
83+
<plugins>
84+
<plugin>
85+
<groupId>org.scala-tools</groupId>
86+
<artifactId>maven-scala-plugin</artifactId>
87+
<executions>
88+
<execution>
89+
<goals>
90+
<goal>compile</goal>
91+
<goal>testCompile</goal>
92+
</goals>
93+
</execution>
94+
</executions>
95+
<configuration>
96+
<scalaVersion>${scala.version}</scalaVersion>
97+
<args>
98+
<arg>-target:jvm-1.5</arg>
99+
</args>
100+
</configuration>
101+
</plugin>
102+
<plugin>
103+
<groupId>org.apache.maven.plugins</groupId>
104+
<artifactId>maven-eclipse-plugin</artifactId>
105+
<configuration>
106+
<downloadSources>true</downloadSources>
107+
<buildcommands>
108+
<buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
109+
</buildcommands>
110+
<additionalProjectnatures>
111+
<projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
112+
</additionalProjectnatures>
113+
<classpathContainers>
114+
<classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
115+
<classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
116+
</classpathContainers>
117+
</configuration>
118+
</plugin>
119+
<plugin>
120+
<groupId>org.apache.maven.plugins</groupId>
121+
<artifactId>maven-surefire-plugin</artifactId>
122+
<version>2.19.1</version>
123+
<configuration>
124+
<skipTests>false</skipTests>
125+
</configuration>
126+
</plugin>
127+
128+
<!-- enable scalatest , after disabling surefire-->
129+
<plugin>
130+
<groupId>org.scalatest</groupId>
131+
<artifactId>scalatest-maven-plugin</artifactId>
132+
<version>1.0</version>
133+
<configuration>
134+
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
135+
<junitxml>.</junitxml>
136+
<filereports>WDF TestSuite.txt</filereports>
137+
</configuration>
138+
<executions>
139+
<execution>
140+
<id>test</id>
141+
<goals>
142+
<goal>test</goal>
143+
</goals>
144+
</execution>
145+
</executions>
146+
</plugin>
147+
</plugins>
148+
</build>
149+
<reporting>
150+
<plugins>
151+
<plugin>
152+
<groupId>org.scala-tools</groupId>
153+
<artifactId>maven-scala-plugin</artifactId>
154+
<configuration>
155+
<scalaVersion>${scala.version}</scalaVersion>
156+
</configuration>
157+
</plugin>
158+
</plugins>
159+
</reporting>
160+
</project>

‎src/main/resources/log4j.properties

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Set everything to be logged to the console
2+
log4j.rootCategory=ERROR, console
3+
log4j.appender.console=org.apache.log4j.ConsoleAppender
4+
log4j.appender.console.target=System.err
5+
log4j.appender.console.layout=org.apache.log4j.PatternLayout
6+
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %C: %m%n
7+
8+
# Settings to quiet third party logs that are too verbose
9+
log4j.logger.org.eclipse.jetty=WARN
10+
log4j.logger.org.apache.spark=WARN
11+
log4j.logger.org.apache.spark.mllib=INFO
12+
log4j.logger.org.spark-project=WARN
13+
log4j.logger.org.spark-project.mllib=INFO
14+
log4j.logger.akka.event=WARN

0 commit comments

Comments
 (0)