VFSClassLoader alternative

modules/local-caching-classloader/.gitignore

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Maven ignores
+/target/
+
+# IDE ignores
+/.settings/
+/.project
+/.classpath
+/.pydevproject
+/.idea
+/*.iml
+/*.ipr
+/*.iws
+/nbproject/
+/nbactions.xml
+/nb-configuration.xml
+/.vscode/
+/.factorypath

modules/local-caching-classloader/README.md

@@ -0,0 +1,104 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Local Caching ClassLoader
+
+The LocalCachingContextClassLoaderFactory is an Accumulo ContextClassLoaderFactory implementation that creates and maintains a
+LocalCachingContext. The `LocalCachingContextClassLoaderFactory.getClassLoader(String)` method expects the method
+argument to be a valid `file`, `hdfs`, `http` or `https` URL to a context definition file.
+
+The context definition file is a JSON formatted file that contains the name of the context, the interval (in seconds) at which
+the context definition file should be monitored, and a list of classpath resources. The LocalCachingContextClassLoaderFactory
+creates the LocalCachingContext based on the initial contents of the context definition file, and updates the classloader
+as changes are noticed based on the monitoring interval. An example of the context definition file is below.
+
+```
+{
+    "contextName": "myContext",
+    "monitorIntervalSeconds": 5,
+    "resources": [
+        {
+            "location": "file:/home/user/ClassLoaderTestA/TestA.jar",
+            "checksum": "a10883244d70d971ec25cbfa69b6f08f"
+        },
+        {
+            "location": "hdfs://localhost:8020/contextB/TestB.jar",
+            "checksum": "a02a3b7026528156fb782dcdecaaa097"
+        },
+        {
+            "location": "http://localhost:80/TestC.jar",
+            "checksum": "f464e66f6d07a41c656e8f4679509215"
+        }
+    ]
+}
+```
+
+## Creating a ContextDefinition file
+
+Users may take advantage of the `ContextDefinition.create` method to construct a ContextDefinition object. This
+will calculate the checksums of the classpath elements. `ContextDefinition.toJson` can be used to serialize the
+ContextDefinition to a file.
+
+## Updating a ContextDefinition file
+
+The LocalCachingContextClassLoaderFactory uses a background thread to fetch the context definition file at the
+specified interval. Users can change the context name, monitor interval, and list of resources. Changes to the
+context name are ignored however as the context cache directory is created using the context name upon initial
+creation. The LocalCachingContextClassLoaderFactory will schedule the next download the of the context
+definition file based on the updated monitor interval, and if the list of resources have changed, then they will
+be downloaded, verified against their checksums, and used to construct a new ClassLoader for the context.
+
+## Local Caching
+
+The property `general.custom.classloader.lcc.cache.dir` is required to be set to a local directory on the host. The
+LocalCachingContext creates a directory at this location for each named context. Each context cache directory
+contains a lock file and a copy of each fetched resource that is named in the context definition file using the format:
+`fileName_checksum`. The lock file is used with Java's `FileChannel.tryLock` to enable exclusive access (on supported
+platforms) to the directory from different processes on the same host.
+
+## Error Handling
+
+If there is an exception in creating the initial classloader, then a ContextClassLoaderException is thrown. If there is
+an exception when updating the classloader, then the exception is logged and the classloader is not updated. Calls
+to `LocalCachingContextClassLoaderFactory.getClassLoader(String)` will return the most recent classloader
+with valid contents. If the checksum of a downloaded resource does not match the checksum in the context definition
+file, then the downloaded version of the file is deleted from the context cache directory so that it can be retried
+at the next interval.
+
+The property `general.custom.classloader.lcc.update.grace.minutes` determines how long the update process
+continues to return the most recent valid classloader when an exception occurs in the background update thread.
+A zero value (default) will cause the most recent valid classloader to be returned. Otherwise, the update thread
+will fail for N minutes, then clear the reference to the classloader internally. This will cause a subsequent
+call to `LocalCachingContextClassLoaderFactory.getClassLoader(String)` to act like the initial call to
+create the classloader and return the exception to the calling code.
+
+## Cleanup
+
+Because the cache directory is shared among multiple processes, and one process can't know what the other processes are doing,
+this class cannot clean up the shared cache directory. It is left to the user to remove unused context cache directories
+and unused old files within a context cache directory.
+
+## Accumulo Configuration
+
+To use this with Accumulo:
+
+  1. Set the following Accumulo site properties: `general.context.class.loader.factory=org.apache.accumulo.classloader.lcc.LocalCachingContextClassLoaderFactory`
+`general.custom.classloader.lcc.cache.dir=file://path/to/some/directory`
+
+  2. Set the following table property: `table.class.loader.context=(file|hdfs|http|https)://path/to/context/definition.json`
+
+

modules/local-caching-classloader/pom.xml

@@ -0,0 +1,227 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      https://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.accumulo</groupId>
+    <artifactId>classloader-extras</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <artifactId>local-caching-classloader</artifactId>
+  <name>classloader-extras-local-caching</name>
+  <properties>
+    <accumulo.build.extraTestArgs>--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.management/java.lang.management=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED --add-opens java.base/java.security=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.stream=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED</accumulo.build.extraTestArgs>
+    <eclipseFormatterStyle>../../src/build/eclipse-codestyle.xml</eclipseFormatterStyle>
+    <failsafe.excludedGroups />
+    <failsafe.failIfNoSpecifiedTests>false</failsafe.failIfNoSpecifiedTests>
+    <failsafe.forkCount>1</failsafe.forkCount>
+    <failsafe.groups />
+    <failsafe.reuseForks>false</failsafe.reuseForks>
+    <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
+    <surefire.excludedGroups />
+    <surefire.failIfNoSpecifiedTests>false</surefire.failIfNoSpecifiedTests>
+    <surefire.forkCount>1</surefire.forkCount>
+    <surefire.groups />
+    <surefire.reuseForks>false</surefire.reuseForks>
+  </properties>
+  <dependencies>
+    <dependency>
+      <!-- needed for build checks, but not for runtime -->
+      <groupId>com.github.spotbugs</groupId>
+      <artifactId>spotbugs-annotations</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.gson</groupId>
+      <artifactId>gson</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <!-- provided by accumulo -->
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-core</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <!-- provided by accumulo -->
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-minicluster</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-minicluster</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j2-impl</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-test-jars</id>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <phase>process-test-resources</phase>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>org.apache.accumulo</groupId>
+                  <artifactId>example-iterators-a</artifactId>
+                  <version>${project.version}</version>
+                  <type>jar</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${project.build.directory}/test-classes/ExampleIteratorsA</outputDirectory>
+                  <destFileName>example-iterators-a.jar</destFileName>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>org.apache.accumulo</groupId>
+                  <artifactId>example-iterators-b</artifactId>
+                  <version>${project.version}</version>
+                  <type>jar</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${project.build.directory}/test-classes/ExampleIteratorsB</outputDirectory>
+                  <destFileName>example-iterators-b.jar</destFileName>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <forkCount>${surefire.forkCount}</forkCount>
+          <reuseForks>${surefire.reuseForks}</reuseForks>
+          <excludedGroups>${surefire.excludedGroups}</excludedGroups>
+          <groups>${surefire.groups}</groups>
+          <systemPropertyVariables combine.children="append">
+            <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+          </systemPropertyVariables>
+          <argLine>${accumulo.build.extraTestArgs}</argLine>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <configuration>
+          <forkCount>${failsafe.forkCount}</forkCount>
+          <reuseForks>${failsafe.reuseForks}</reuseForks>
+          <excludedGroups>${failsafe.excludedGroups}</excludedGroups>
+          <groups>${failsafe.groups}</groups>
+          <systemPropertyVariables combine.children="append">
+            <accumulo.it.uniq.test.dir>${accumulo.it.uniq.test.dir}</accumulo.it.uniq.test.dir>
+            <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+          </systemPropertyVariables>
+          <argLine>${accumulo.build.extraTestArgs}</argLine>
+          <trimStackTrace>false</trimStackTrace>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>build-test-jars</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>process-test-classes</phase>
+            <configuration>
+              <executable>src/test/shell/makeTestJars.sh</executable>
+            </configuration>
+          </execution>
+          <execution>
+            <id>build-helloworld-jars</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>process-test-classes</phase>
+            <configuration>
+              <executable>src/test/shell/makeHelloWorldJars.sh</executable>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>