KAFKA-139 cross-compile multiple Scala versions and upgrade to SBT 0.12.1 patch by Derek Chen-Becker reviewed by Joe Stein

.gitignore

@@ -12,3 +12,5 @@ project/sbt_project_definition.iml
 *~
 *#
 .#*
+rat.out
+TAGS

README.md

@@ -9,39 +9,41 @@ It is designed to support the following
 
 Kafka is aimed at providing a publish-subscribe solution that can handle all activity stream data and processing on a consumer-scale web site. This kind of activity (page views, searches, and other user actions) are a key ingredient in many of the social feature on the modern web. This data is typically handled by "logging" and ad hoc log aggregation solutions due to the throughput requirements. This kind of ad hoc solution is a viable solution to providing logging data to an offline analysis system like Hadoop, but is very limiting for building real-time processing. Kafka aims to unify offline and online processing by providing a mechanism for parallel load into Hadoop as well as the ability to partition real-time consumption over a cluster of machines.
 
-See our [web site](http://incubator.apache.org/kafka/) for more details on the project.
+See our [web site](http://kafka.apache.org/) for more details on the project.
 
 ## Contribution ##
 
-Kafka is a new project, and we are interested in building the community; we would welcome any thoughts or [patches](https://issues.apache.org/jira/browse/KAFKA). You can reach us [on the Apache mailing lists](http://incubator.apache.org/kafka/contact.html).
+Kafka is a new project, and we are interested in building the community; we would welcome any thoughts or [patches](https://issues.apache.org/jira/browse/KAFKA). You can reach us [on the Apache mailing lists](http://kafka.apache.org/contact.html).
 
-The Kafka code is available from svn or a read only git mirror:
+The Kafka code is available from:
- * svn co http://svn.apache.org/repos/asf/incubator/kafka/trunk kafka
+ * git clone http://git-wip-us.apache.org/repos/asf/kafka.git kafka
- * git clone git://git.apache.org/kafka.git
 
-To build: 
+To contribute you can follow:
+ * https://cwiki.apache.org/confluence/display/KAFKA/Git+Workflow
 
-1. ./sbt
+To build for all supported versions of Scala: 
-2. update - This downloads all the dependencies for all sub projects
-3. package - This will compile all sub projects and creates all the jars
 
-Here are some useful sbt commands, to be executed at the sbt command prompt (./sbt) -
+1. ./sbt +package
 
-actions : Lists all the sbt commands and their descriptions
+To build for a particular version of Scala (either 2.8.0, 2.8.2, 2.9.1 or 2.9.2): 
+
+1. ./sbt "++2.8.0 package" *or* ./sbt "++2.8.2 package" *or* ./sbt "++2.9.1 package" *or* ./sbt "++2.9.2 package"
+
+Here are some useful sbt commands, to be executed at the sbt command prompt (./sbt). Prefixing with "++<version> " runs the
+command for a specific Scala version, prefixing with "+" will perform the action for all versions of Scala, and no prefix
+runs the command for the default (2.8.0) version of Scala. -
+
+tasks : Lists all the sbt commands and their descriptions
 
 clean : Deletes all generated files (the target directory).
 
-clean-cache : Deletes the cache of artifacts downloaded for automatically managed dependencies.
-
-clean-lib : Deletes the managed library directory.
-
 compile : Compile all the sub projects, but not create the jars
 
 test : Run all unit tests in all sub projects
 
 release-zip : Create all the jars, run unit tests and create a deployable release zip
 
-package-all: Creates jars for src, test, docs etc
+package: Creates jars for src, test, docs etc
 
 projects : List all the sub projects 
 
@@ -53,4 +55,5 @@ test-only package.test.TestName : Runs only the specified test in the current su
 
 run : Provides options to run any of the classes that have a main method. For example, you can switch to project java-examples, and run the examples there by executing "project java-examples" followed by "run" 
 
+For more details please see the [SBT documentation](https://github.com/harrah/xsbt/wiki)
 

bin/run-rat.sh

@@ -23,7 +23,7 @@ else
   JAVA="$JAVA_HOME/bin/java"
 fi
 
-rat_command="$JAVA -jar $base_dir/lib_managed/scala_2.8.0/compile/apache-rat-0.8.jar --dir $base_dir "
+rat_command="$JAVA -jar $base_dir/lib/apache-rat-0.8.jar --dir $base_dir "
 
 for f in $(cat $rat_excludes_file);
 do

contrib/hadoop-consumer/build.sbt

@@ -0,0 +1 @@
+crossPaths := false

contrib/hadoop-producer/build.sbt

@@ -0,0 +1 @@
+crossPaths := false

core/build.sbt

@@ -0,0 +1,24 @@
+import sbt._
+import Keys._
+
+name := "kafka"
+
+resolvers ++= Seq(
+  "SonaType ScalaTest repo" at "https://oss.sonatype.org/content/groups/public/org/scalatest/"
+)
+
+libraryDependencies ++= Seq(
+  "org.apache.zookeeper"  % "zookeeper"   % "3.3.4",
+  "com.github.sgroschupf" % "zkclient"    % "0.1",
+  "org.xerial.snappy"     % "snappy-java" % "1.0.4.1",
+  "org.easymock"          % "easymock"    % "3.0" % "test",
+  "junit"                 % "junit"       % "4.1" % "test"
+)
+
+libraryDependencies <<= (scalaVersion, libraryDependencies) { (sv, deps) =>
+  deps :+ (sv match {
+    case "2.8.0" => "org.scalatest" %  "scalatest" % "1.2" % "test"
+    case _       => "org.scalatest" %% "scalatest" % "1.8" % "test"
+  })
+}
+

examples/build.sbt

@@ -0,0 +1,3 @@
+name := "kafka-java-examples"
+
+crossPaths := false

project/Build.scala

@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt._
+import Keys._
+import java.io.File
+
+import scala.xml.{Node, Elem}
+import scala.xml.transform.{RewriteRule, RuleTransformer}
+
+object KafkaBuild extends Build {
+  val commonSettings = Seq(
+    version := "0.8-SNAPSHOT",
+    organization := "org.apache",
+    scalacOptions ++= Seq("-deprecation", "-unchecked", "-g:none"),
+    crossScalaVersions := Seq("2.8.0","2.8.2", "2.9.1", "2.9.2"),
+    scalaVersion := "2.8.0",
+    javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.5"),
+    parallelExecution in Test := false, // Prevent tests from overrunning each other
+    libraryDependencies ++= Seq(
+      "log4j"                 % "log4j"        % "1.2.15",
+      "net.sf.jopt-simple"    % "jopt-simple"  % "3.2",
+      "org.slf4j"             % "slf4j-simple" % "latest.release"
+    ),
+    // The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required
+    // some dependencies on various sun and javax packages.
+    ivyXML := <dependencies>
+        <exclude module="javax"/>
+        <exclude module="jmxri"/>
+        <exclude module="jmxtools"/>
+        <exclude module="mail"/>
+        <exclude module="jms"/>
+        <dependency org="org.apache.zookeeper" name="zookeeper" rev="3.3.4">
+          <exclude org="log4j" module="log4j"/>
+          <exclude org="jline" module="jline"/>
+        </dependency>
+      </dependencies>
+  )
+
+  val hadoopSettings = Seq(
+    javacOptions ++= Seq("-Xlint:deprecation"),
+    libraryDependencies ++= Seq(
+      "org.apache.avro"      % "avro"               % "1.4.0",
+      "org.apache.pig"       % "pig"                % "0.8.0",
+      "commons-logging"      % "commons-logging"    % "1.0.4",
+      "org.codehaus.jackson" % "jackson-core-asl"   % "1.5.5",
+      "org.codehaus.jackson" % "jackson-mapper-asl" % "1.5.5",
+      "org.apache.hadoop"    % "hadoop-core"        % "0.20.2"
+    ),
+    ivyXML := 
+       <dependencies>
+         <exclude module="netty"/>
+         <exclude module="javax"/>
+         <exclude module="jmxri"/>
+         <exclude module="jmxtools"/>
+         <exclude module="mail"/>
+         <exclude module="jms"/>
+         <dependency org="org.apache.hadoop" name="hadoop-core" rev="0.20.2">
+           <exclude org="junit" module="junit"/>
+         </dependency>
+         <dependency org="org.apache.pig" name="pig" rev="0.8.0">
+           <exclude org="junit" module="junit"/>
+         </dependency>
+       </dependencies>
+  )
+
+  val coreSettings = Seq(
+    pomPostProcess := { (pom: Node) => MetricsDepAdder(ZkClientDepAdder(pom)) }
+  )
+
+  val runRat = TaskKey[Unit]("run-rat-task", "Runs Apache rat on Kafka")
+  val runRatTask = runRat := {
+    "bin/run-rat.sh" !
+  }
+
+  lazy val kafka    = Project(id = "Kafka", base = file(".")).aggregate(core, examples, contrib).settings((commonSettings ++ runRatTask): _*)
+  lazy val core     = Project(id = "core", base = file("core")).settings(commonSettings: _*).settings(coreSettings: _*)
+  lazy val examples = Project(id = "java-examples", base = file("examples")).settings(commonSettings :_*) dependsOn (core)
+  lazy val perf     = Project(id = "perf", base = file("perf")).settings((Seq(name := "kafka-perf") ++ commonSettings):_*) dependsOn (core)
+
+  lazy val contrib        = Project(id = "contrib", base = file("contrib")).aggregate(hadoopProducer, hadoopConsumer).settings(commonSettings :_*)
+  lazy val hadoopProducer = Project(id = "hadoop-producer", base = file("contrib/hadoop-producer")).settings(hadoopSettings ++ commonSettings: _*) dependsOn (core)
+  lazy val hadoopConsumer = Project(id = "hadoop-consumer", base = file("contrib/hadoop-consumer")).settings(hadoopSettings ++ commonSettings: _*) dependsOn (core)
+
+
+  // POM Tweaking for core:
+  def zkClientDep =
+    <dependency>
+      <groupId>zkclient</groupId>
+      <artifactId>zkclient</artifactId>
+      <version>20120522</version>
+      <scope>compile</scope>
+    </dependency>
+
+  def metricsDeps =
+    <dependencies>
+      <dependency>
+        <groupId>com.yammer.metrics</groupId>
+        <artifactId>metrics-core</artifactId>
+        <version>3.0.0-c0c8be71</version>
+        <scope>compile</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.yammer.metrics</groupId>
+        <artifactId>metrics-annotations</artifactId>
+        <version>3.0.0-c0c8be71</version>
+        <scope>compile</scope>
+      </dependency>
+    </dependencies>
+
+  object ZkClientDepAdder extends RuleTransformer(new RewriteRule() {
+    override def transform(node: Node): Seq[Node] = node match {
+      case Elem(prefix, "dependencies", attribs, scope, deps @ _*) => {
+        Elem(prefix, "dependencies", attribs, scope, deps ++ zkClientDep:_*)
+      }
+      case other => other
+    }
+  })
+
+  object MetricsDepAdder extends RuleTransformer(new RewriteRule() {
+    override def transform(node: Node): Seq[Node] = node match {
+      case Elem(prefix, "dependencies", attribs, scope, deps @ _*) => {
+        Elem(prefix, "dependencies", attribs, scope, deps ++ metricsDeps:_*)
+      }
+      case other => other
+    }
+  })
+
+}

project/build.properties

@@ -14,11 +14,4 @@
 # limitations under the License.
 #Project properties
 #Mon Feb 28 11:55:49 PST 2011
-project.name=Kafka
+sbt.version=0.12.1
-sbt.version=0.7.5
-project.version=0.8.0-SNAPSHOT
-build.scala.versions=2.8.0
-contrib.root.dir=contrib
-lib.dir=lib
-target.dir=target/scala_2.8.0
-dist.dir=dist

project/plugins.sbt

@@ -0,0 +1,5 @@
+resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
+
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.5")
+
+addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.2.0")

project/plugins/Plugins.scala

@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt._
-
-class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
-  val repo = "GH-pages repo" at "http://mpeltonen.github.com/maven/"
-  val idea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.1-SNAPSHOT"
-}