diff --git a/DeployUtils/README.md b/DeployUtils/README.md new file mode 100644 index 0000000..d791271 --- /dev/null +++ b/DeployUtils/README.md @@ -0,0 +1,128 @@ +DeployUtils +==== +Deploy to Embedded Targets in both Java and C++. + +For all projects, you can define deployment targets and artifacts. The deploy process works over SSH/SFTP and +is extremely quick. + +For the previous functionality for native library building, see edu.wpi.first.NativeUtils + +Commands: +`gradlew deploy` will deploy all artifacts +`gradlew deployStandalone` will deploy all artifacts marked for standalone use with `allowStandaloneDeploy` +`gradlew deploy` will deploy only the specified artifact to the specified target + +Properties: +`gradlew deploy -Pdeploy-dirty` will skip the cache check and force redeployment of all files +`gradlew deploy -Pdeploy-dry` will do a 'dry run' (will not connect or deploy to target, instead only printing to console) + +## Installing plugin +Include the following in your `build.gradle` +```gradle +plugins { + id "edu.wpi.first.DeployUtils" version "" +} +``` + +See [https://plugins.gradle.org/plugin/edu.wpi.first.DeployUtils](https://plugins.gradle.org/plugin/edu.first.wpi.DeployUtils) for the latest version + +## Spec + +```gradle + +// DSL (all properties optional unless stated as required) +deploy { + targets { + myTarget(getTargetTypeClass('RemoteTarget')) { // name is first, parameter to getTargetTypeClass is type + directory = '/home/myuser' // The root directory to start deploying to. Default: user home + maxChannels = 1 // The number of channels to open on the target (how many files / commands to run at the same time). Default: 1 + timeout = 3 // Timeout to use when connecting to target. Default: 3 (seconds) + failOnMissing = true // Should the build fail if the target can't be found? Default: true + + locations { + ssh(getLocationTypeClass('SshDeployLocation')) { + address = "mytarget.local" // Required. The address to try + user = 'myuser' // Required. The user to login as + password = '' // The password for the user. Default: blank (empty) string + ipv6 = false // Are IPv6 addresses permitted? Default: false + } + } + + // Artifacts are specific per target + artifacts { + // COMMON PROPERTIES FOR ALL ARTIFACTS // + all { + directory = 'mydir' // Subdirectory to use. Relative to target directory + + onlyIf = { execute('echo Hi').result == 'Hi' } // Check closure for artifact. Will not deploy if evaluates to false + + predeploy << { execute 'echo Pre' } // After onlyIf, but before deploy logic + postdeploy << { execute 'echo Post' } // After this artifact's deploy logic + + disabled = true // Disable this artifact. Default: false. + + dependsOn('someTask') // Make this artifact depend on a task, both standalone and main deploy tasks + + dependsOnForDeployTask('someTask') // Make main artifact deploy task only depend on task + + dependsOnForStandaloneDeployTask('someTask') // Make standalone artifact deploy task only depend on task + } + // END COMMON // + + myFileArtifact(getArtifactTypeClass('FileArtifact)) { + file = file('myFile') // Set the file to deploy. Required. + filename = 'myFile.dat' // Set the filename to deploy to. Default: same name as file + } + + // FileCollectionArtifact is a flat collection of files - directory structure is not preserved + myFileCollectionArtifact(getArtifactTypeClass('FileCollectionArtifact)) { + files = fileTree(dir: 'myDir') // Required. Set the filecollection (e.g. filetree, files, etc) to deploy + } + + // FileTreeArtifact is like a FileCollectionArtifact, but the directory structure is preserved + myFileTreeArtifact(getArtifactTypeClass('FileTreeArtifact)) { + files = fileTree(dir: 'mydir') // Required. Set the fileTree (e.g. filetree, ziptree) to deploy + } + + myCommandArtifact(getArtifactTypeClass('CommandArtifact)) { + command = 'echo Hello' // The command to run. Required. + // Output will be stored in 'result' after execution + } + + // JavaArtifact inherits from FileArtifact + myJavaArtifact(getArtifactTypeClass('JavaArtifact)) { + // The binary to deploy is not configured by default. To configure, + // assign the exectuable property to the binary you want to run. + // See below for how to do this. + // High level plugins can provide an easier way to do this. + } + + myNativeArtifact(getArtifactTypeClass('NativeExecutableArtifact)) { + // The binary to deploy is not configured by default. To configure, + // assign the exectuable property to the binary you want to run. + // See below for how to do this. + // High level plugins can provide an easier way to do this. + } + } + } + } +} + +// For Java +deploy.targets.myTarget.artifacts.myJavaArtifact.jarTask = jar // Assuming you have a standard 'java' plugin + +// For Native Code +model { + components { + my_program(NativeExecutableSpec) { + binaries.all { + // Filter to binary you want to deploy here. + // For instace + if (it.targetPlatform.name == 'SomeCrossBuild' && it.buildType.name == 'debug') { + deploy.targets.myTarget.artifacts.myNativeArtifact.binary = it + } + } + } + } +} +``` diff --git a/DeployUtils/build.gradle b/DeployUtils/build.gradle new file mode 100644 index 0000000..59bb992 --- /dev/null +++ b/DeployUtils/build.gradle @@ -0,0 +1,60 @@ +plugins { + id 'com.gradle.plugin-publish' + id 'java-gradle-plugin' + id 'groovy' + id 'maven-publish' + id 'idea' +} + +repositories { + maven { + url "https://plugins.gradle.org/m2/" + } + mavenLocal() + mavenCentral() +} + +dependencies { + api 'com.jcraft:jsch:0.1.55' + api 'com.google.code.gson:gson:2.8.6' + + testImplementation('org.spockframework:spock-core:2.0-M4-groovy-3.0') { + exclude group: 'org.codehaus.groovy' + } + testImplementation("org.junit.jupiter:junit-jupiter-api:5.7.1") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") + testImplementation('cglib:cglib-nodep:2.2') + testImplementation('org.objenesis:objenesis:1.2') + testImplementation gradleTestKit() +} + +tasks.withType(Test).configureEach { + useJUnitPlatform() +} + +base { + archivesName = "DeployUtils" +} + +gradlePlugin { + website = 'https://github.com/wpilibsuite/native-utils' + vcsUrl = 'https://github.com/wpilibsuite/native-utils' + plugins { + DeployUtils { + id = 'edu.wpi.first.DeployUtils' + displayName = 'DeployUtils' + implementationClass = 'edu.wpi.first.deployutils.DeployUtils' + description = 'Additions to the model-based DSL for deploying Java and Native projects to remote targets' + tags = ['remote', 'target', 'deploy', 'java', 'native'] + } + } +} + +tasks.withType(JavaCompile) { + options.compilerArgs << '-Xlint:unchecked' + options.deprecation = true +} + +tasks.withType(Javadoc) { + options.addBooleanOption('Xdoclint:all,-missing', true) +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/ActionWrapper.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/ActionWrapper.java new file mode 100644 index 0000000..1035851 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/ActionWrapper.java @@ -0,0 +1,18 @@ +package edu.wpi.first.deployutils; + +import org.gradle.api.Action; + +import groovy.lang.Closure; + +public class ActionWrapper implements Action { + private final Closure closure; + + public ActionWrapper(Closure closure) { + this.closure = closure; + } + + @Override + public void execute(T t) { + ClosureUtils.delegateCall(t, closure); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/ClosureUtils.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/ClosureUtils.java new file mode 100644 index 0000000..f4f4e2f --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/ClosureUtils.java @@ -0,0 +1,22 @@ +package edu.wpi.first.deployutils; + +import groovy.lang.Closure; + +public class ClosureUtils { + public static Object delegateCall(Object object, Closure closure, Object... args) { + closure.setResolveStrategy(Closure.DELEGATE_FIRST); + closure.setDelegate(object); + Object[] passArgs = new Object[args.length + 1]; + passArgs[0] = object; + for (int i = 0; i < args.length; i++) { + passArgs[i + 1] = args[i]; + } + return closure.call(passArgs); + } + + public static Object delegateCall(Object object, Closure closure) { + closure.setResolveStrategy(Closure.DELEGATE_FIRST); + closure.setDelegate(object); + return closure.call(object); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/DeployUtils.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/DeployUtils.java new file mode 100644 index 0000000..6996c73 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/DeployUtils.java @@ -0,0 +1,34 @@ +package edu.wpi.first.deployutils; + +import com.jcraft.jsch.JSch; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +import edu.wpi.first.deployutils.deploy.DeployPlugin; +import edu.wpi.first.deployutils.log.ETLoggerFactory; + +public class DeployUtils implements Plugin { + + @Override + public void apply(Project project) { + + ETLoggerFactory.INSTANCE.addColorOutput(project); + + project.getPluginManager().apply(DeployPlugin.class); + } + + private static JSch jsch; + public static JSch getJsch() { + if (jsch == null) jsch = new JSch(); + return jsch; + } + + public static boolean isDryRun(Project project) { + return project.hasProperty("deploy-dry"); + } + + public static boolean isSkipCache(Project project) { + return project.hasProperty("deploy-dirty"); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/PathUtils.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/PathUtils.java new file mode 100644 index 0000000..05fcb22 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/PathUtils.java @@ -0,0 +1,28 @@ +package edu.wpi.first.deployutils; + +import java.util.Stack; + +public class PathUtils { + public static String combine(String root, String relative) { + return normalize(relative == null ? root : join(root, relative)); + } + + public static String join(String root, String relative) { + if (relative.startsWith("/")) return relative; + if (root.charAt(root.length() - 1) != '/') root += '/'; + return root += relative; + } + + public static String normalize(String filepath) { + String[] strings = filepath.split("/"); + Stack s = new Stack<>(); + for (String str : strings) { + if (str.trim().equals("..")) { + s.pop(); + } else { + s.push(str); + } + } + return String.join("/", s); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/PredicateWrapper.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/PredicateWrapper.java new file mode 100644 index 0000000..6d7728e --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/PredicateWrapper.java @@ -0,0 +1,19 @@ +package edu.wpi.first.deployutils; + +import java.util.function.Predicate; + +import groovy.lang.Closure; + +public class PredicateWrapper implements Predicate { + private final Closure closure; + + public PredicateWrapper(Closure closure) { + this.closure = closure; + } + + @Override + public boolean test(T t) { + return ((Boolean)ClosureUtils.delegateCall(t, closure)).booleanValue(); + } +} + diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/CommandDeployResult.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/CommandDeployResult.java new file mode 100644 index 0000000..2c7b68e --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/CommandDeployResult.java @@ -0,0 +1,51 @@ +package edu.wpi.first.deployutils.deploy; + +import java.util.Objects; + +public class CommandDeployResult { + private final String command; + + public final String getCommand() { + return command; + } + + private final String result; + + public final String getResult() { + return result; + } + + private final int exitCode; + + public final int getExitCode() { + return exitCode; + } + + public CommandDeployResult(String command, String result, int exitCode) { + this.command = command; + this.result = result; + this.exitCode = exitCode; + } + + @Override + public int hashCode() { + return Objects.hash(command, result, exitCode); + } + + @Override + public boolean equals(Object other) { + if (other instanceof CommandDeployResult) { + CommandDeployResult cdr = (CommandDeployResult)other; + return Objects.equals(command, cdr.command) && + Objects.equals(result, cdr.result) && + exitCode == cdr.exitCode; + } + return false; + } + + @Override + public String toString() { + return "CommandDeployResult(" + getCommand() + ", " + getResult() + ", " + getExitCode() + ")"; + } + +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/DeployExtension.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/DeployExtension.java new file mode 100644 index 0000000..b35424a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/DeployExtension.java @@ -0,0 +1,181 @@ +package edu.wpi.first.deployutils.deploy; + +import java.util.Set; +import java.util.concurrent.Callable; + +import javax.inject.Inject; + +import org.gradle.api.ExtensiblePolymorphicDomainObjectContainer; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.diagnostics.TaskReportTask; + +import edu.wpi.first.deployutils.deploy.artifact.ActionArtifact; +import edu.wpi.first.deployutils.deploy.artifact.Artifact; +import edu.wpi.first.deployutils.deploy.artifact.CacheableArtifact; +import edu.wpi.first.deployutils.deploy.artifact.CommandArtifact; +import edu.wpi.first.deployutils.deploy.artifact.FileArtifact; +import edu.wpi.first.deployutils.deploy.artifact.FileCollectionArtifact; +import edu.wpi.first.deployutils.deploy.artifact.FileTreeArtifact; +import edu.wpi.first.deployutils.deploy.artifact.JavaArtifact; +import edu.wpi.first.deployutils.deploy.artifact.MavenArtifact; +import edu.wpi.first.deployutils.deploy.artifact.MultiCommandArtifact; +import edu.wpi.first.deployutils.deploy.artifact.NativeExecutableArtifact; +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.cache.Md5FileCacheMethod; +import edu.wpi.first.deployutils.deploy.cache.Md5SumCacheMethod; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; +import edu.wpi.first.deployutils.deploy.target.location.SshDeployLocation; +import org.gradle.api.internal.PolymorphicDomainObjectContainerInternal; + +public class DeployExtension { + + private final TaskProvider deployTask; + private final TaskProvider standaloneDeployTask; + private final TaskProvider listTypeClassesTask; + + private final ExtensiblePolymorphicDomainObjectContainer targets; + private final ExtensiblePolymorphicDomainObjectContainer cache; + private final Provider storageServiceProvider; + + public TaskProvider getListTypeClassesTask() { + return listTypeClassesTask; + } + + public Provider getStorageServiceProvider() { + return storageServiceProvider; + } + + public ExtensiblePolymorphicDomainObjectContainer getTargets() { + return targets; + } + + public ExtensiblePolymorphicDomainObjectContainer getCache() { + return cache; + } + + public Class getTargetTypeClass(String name) { + @SuppressWarnings("unchecked") + PolymorphicDomainObjectContainerInternal internalTargets = + (PolymorphicDomainObjectContainerInternal) targets; + Set> targetTypeSet = internalTargets.getCreateableTypes(); + for (Class targetType : targetTypeSet) { + if (targetType.getSimpleName().equals(name)) { + return targetType; + } + } + return null; + } + + public Class getCacheTypeClass(String name) { + @SuppressWarnings("unchecked") + PolymorphicDomainObjectContainerInternal internalCache = + (PolymorphicDomainObjectContainerInternal) cache; + Set> cacheTypeSet = internalCache.getCreateableTypes(); + for (Class cacheType : cacheTypeSet) { + if (cacheType.getSimpleName().equals(name)) { + return cacheType; + } + } + return null; + } + + public TaskProvider getDeployTask() { + return deployTask; + } + + public TaskProvider getStandaloneDeployTask() { + return standaloneDeployTask; + } + + private void configureTargetTypes(RemoteTarget target) { + ObjectFactory objects = target.getProject().getObjects(); + ExtensiblePolymorphicDomainObjectContainer locations = target.getLocations(); + ExtensiblePolymorphicDomainObjectContainer artifacts = target.getArtifacts(); + + NamedObjectFactory.registerType(NativeExecutableArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(FileArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(JavaArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(ActionArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(FileCollectionArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(FileTreeArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(MavenArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(MultiCommandArtifact.class, artifacts, target, objects); + NamedObjectFactory.registerType(CommandArtifact.class, artifacts, target, objects); + + NamedObjectFactory.registerType(SshDeployLocation.class, locations, target, objects); + } + + @Inject + public DeployExtension(Project project, ObjectFactory objects) { + + storageServiceProvider = project.getGradle().getSharedServices().registerIfAbsent("deployPluginStorageService", StorageService.class, spec -> {}); + + targets = objects.polymorphicDomainObjectContainer(RemoteTarget.class); + cache = objects.polymorphicDomainObjectContainer(CacheMethod.class); + + cache.registerFactory(Md5SumCacheMethod.class, name -> { + return objects.newInstance(Md5SumCacheMethod.class, name); + }); + + cache.registerFactory(Md5FileCacheMethod.class, name -> { + return objects.newInstance(Md5FileCacheMethod.class, name); + }); + + cache.register("md5sum", Md5SumCacheMethod.class); + + targets.registerFactory(RemoteTarget.class, name -> { + return objects.newInstance(RemoteTarget.class, name, project, this); + }); + + // Empty all forces all registered items to be instantly resolved. + // Without this, any registered tasks will crash when deploy is called + // Also resolve all inner artifacts for same reason + targets.all(x -> { + configureTargetTypes(x); + x.getArtifacts().all(y -> { + if (y instanceof CacheableArtifact) { + Property cm = ((CacheableArtifact)y).getCacheMethod(); + if (!cm.isPresent()) { + cm.set(cache.getByName("md5sum")); + } + } + }); + }); + + deployTask = project.getTasks().register("deploy", task -> { + task.setGroup("DeployUtils"); + task.setDescription("Deploy all artifacts on all targets"); + targets.all(x -> { + task.dependsOn(x.getDeployTask()); + }); + }); + + standaloneDeployTask = project.getTasks().register("deployStandalone", task -> { + task.setGroup("DeployUtils"); + task.setDescription("Deploy all artifacts on all targets"); + targets.all(x -> { + task.dependsOn(x.getStandaloneDeployTask()); + }); + }); + + listTypeClassesTask = project.getTasks().register("listTypeClasses", ListBaseTypeClassesTask.class, task -> { + task.setGroup("DeployUtils"); + task.setDescription("Lists all type classes for targets and cache methods"); + task.setExtension(this); + }); + + project.getTasks().withType(TaskReportTask.class).configureEach(t -> { + Callable enumTask = () -> { + targets.all(x -> x.getArtifacts().all(y -> {})); + return null; + }; + t.dependsOn(enumTask); + }); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/DeployPlugin.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/DeployPlugin.java new file mode 100644 index 0000000..7e81b14 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/DeployPlugin.java @@ -0,0 +1,14 @@ +package edu.wpi.first.deployutils.deploy; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.language.base.plugins.ComponentModelBasePlugin; + +public class DeployPlugin implements Plugin { + @Override + public void apply(Project project) { + project.getPluginManager().apply(ComponentModelBasePlugin.class); + + project.getExtensions().create("deploy", DeployExtension.class, project); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/ListBaseTypeClassesTask.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/ListBaseTypeClassesTask.java new file mode 100644 index 0000000..7a92ce1 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/ListBaseTypeClassesTask.java @@ -0,0 +1,47 @@ +package edu.wpi.first.deployutils.deploy; + +import java.util.Set; + +import org.gradle.api.DefaultTask; +import org.gradle.api.internal.PolymorphicDomainObjectContainerInternal; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.UntrackedTask; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +@UntrackedTask(because = "Helper task") +public class ListBaseTypeClassesTask extends DefaultTask { + private DeployExtension extension; + + public void setExtension(DeployExtension extension) { + this.extension = extension; + } + + @SuppressWarnings("unchecked") + @TaskAction + public void execute() { + + getLogger().lifecycle("Type classes for targets and cache methods"); + + getLogger().lifecycle("Target Type Classes (getTargetTypeClass):"); + + PolymorphicDomainObjectContainerInternal internalTargets = + (PolymorphicDomainObjectContainerInternal) extension.getTargets(); + Set> targetTypeSet = internalTargets.getCreateableTypes(); + for (Class targetType : targetTypeSet) { + getLogger().lifecycle("\t{}", targetType.getSimpleName()); + } + + getLogger().lifecycle(""); + getLogger().lifecycle("Cache Method Type Classes (getCacheTypeClass):"); + + PolymorphicDomainObjectContainerInternal internalCache = + (PolymorphicDomainObjectContainerInternal) extension.getCache(); + Set> cacheTypeSet = internalCache.getCreateableTypes(); + for (Class cacheType : cacheTypeSet) { + getLogger().lifecycle("\t{}", cacheType.getSimpleName()); + } + + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/NamedObjectFactory.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/NamedObjectFactory.java new file mode 100644 index 0000000..994f2bb --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/NamedObjectFactory.java @@ -0,0 +1,28 @@ +package edu.wpi.first.deployutils.deploy; + +import org.gradle.api.ExtensiblePolymorphicDomainObjectContainer; +import org.gradle.api.NamedDomainObjectFactory; +import org.gradle.api.model.ObjectFactory; + +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class NamedObjectFactory implements NamedDomainObjectFactory { + private final ObjectFactory objects; + private final RemoteTarget target; + private final Class cls; + + @Override + public T create(String name) { + return objects.newInstance(cls, name, target); + } + + public NamedObjectFactory(ObjectFactory objects, RemoteTarget target, Class cls) { + this.objects = objects; + this.target = target; + this.cls = cls; + } + + public static void registerType(Class cls, ExtensiblePolymorphicDomainObjectContainer registry, RemoteTarget remote, ObjectFactory objects) { + registry.registerFactory(cls, new NamedObjectFactory(objects, remote, cls)); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/StorageService.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/StorageService.java new file mode 100644 index 0000000..3d9fc13 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/StorageService.java @@ -0,0 +1,92 @@ +package edu.wpi.first.deployutils.deploy; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import javax.inject.Inject; + +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; + +import edu.wpi.first.deployutils.deploy.artifact.Artifact; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.sessions.SessionController; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public abstract class StorageService implements BuildService, AutoCloseable { + + public static class DeployStorage { + public final DeployContext context; + public final Artifact artifact; + public DeployStorage(DeployContext context, Artifact artifact) { + this.context = context; + this.artifact = artifact; + } + } + + public static class DiscoveryStorage { + public final RemoteTarget target; + public DiscoveryStorage(RemoteTarget target, Consumer contextSet) { + this.target = target; + this.contextSet = contextSet; + } + public final Consumer contextSet; + } + + private final AtomicInteger hashIndex; + private final ConcurrentMap deployerStorage; + private final ConcurrentMap discoveryStorage; + private final List sessions; + + @Inject + public StorageService() { + hashIndex = new AtomicInteger(0); + deployerStorage = new ConcurrentHashMap<>(); + discoveryStorage = new ConcurrentHashMap<>(); + sessions = Collections.synchronizedList(new ArrayList<>()); + } + + public int submitDeployStorage(DeployContext context, Artifact artifact) { + DeployStorage ds = new DeployStorage(context, artifact); + int idx = hashIndex.getAndIncrement(); + deployerStorage.put(idx, ds); + return idx; + } + + public DeployStorage getDeployStorage(int idx) { + return deployerStorage.get(idx); + } + + public int submitDiscoveryStorage(RemoteTarget target, Consumer cb) { + DiscoveryStorage stg = new DiscoveryStorage(target, cb); + int idx = hashIndex.getAndIncrement(); + discoveryStorage.put(idx, stg); + return idx; + } + + public DiscoveryStorage getDiscoveryStorage(int idx) { + return discoveryStorage.get(idx); + } + + public void addSessionForCleanup(SessionController session) { + sessions.add(session); + } + + @Override + public void close() { + deployerStorage.clear(); + discoveryStorage.clear(); + for (SessionController sessionController : sessions) { + try { + sessionController.close(); + } catch (Exception e) { + } + } + sessions.clear(); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/AbstractArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/AbstractArtifact.java new file mode 100644 index 0000000..c7d7a22 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/AbstractArtifact.java @@ -0,0 +1,166 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.util.List; +import java.util.function.Predicate; + +import javax.inject.Inject; + +import org.gradle.api.Action; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.TaskProvider; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; + +public abstract class AbstractArtifact implements Artifact { + private final String name; + private final RemoteTarget target; + private final TaskProvider deployTask; + private final TaskProvider standaloneDeployTask; + + private boolean disabled = false; + + private final Property directory; + private final List> predeploy = new WrappedArrayList<>(); + private final List> postdeploy = new WrappedArrayList<>(); + private final List> preWorkerThread = new WrappedArrayList<>(); + private Predicate onlyIf = null; + + @Inject + public AbstractArtifact(String name, RemoteTarget target) { + this.name = name; + directory = target.getProject().getObjects().property(String.class); + directory.set(""); + this.target = target; + + deployTask = target.getProject().getTasks().register("deploy" + name + target.getName(), ArtifactDeployTask.class, task -> { + task.getArtifact().set(this); + task.getTarget().set(target); + task.setGroup("DeployUtils"); + task.setDescription("Deploys " + name + " to " + target.getName()); + + task.dependsOn(target.getTargetDiscoveryTask()); + task.getStorageService().set(target.getStorageServiceProvider()); + task.usesService(target.getStorageServiceProvider()); + }); + target.getDeployTask().configure(x -> x.dependsOn(deployTask)); + + standaloneDeployTask = target.getProject().getTasks().register("deployStandalone" + name + target.getName(), ArtifactDeployTask.class, task -> { + task.getArtifact().set(this); + task.getTarget().set(target); + task.setGroup("DeployUtils"); + task.setDescription("Deploys " + name + " to " + target.getName() + " as Standalone"); + + task.dependsOn(target.getTargetDiscoveryTask()); + task.getStorageService().set(target.getStorageServiceProvider()); + task.usesService(target.getStorageServiceProvider()); + }); + } + + @Override + public TaskProvider getStandaloneDeployTask() { + return deployTask; + } + + @Override + public void allowStandaloneDeploy() { + target.getStandaloneDeployTask().configure(x -> x.dependsOn(standaloneDeployTask)); + } + + @Override + public TaskProvider getDeployTask() { + return deployTask; + } + + @Override + public RemoteTarget getTarget() { + return target; + } + + @Override + public String getName() { + return name; + } + + @Override + public void dependsOn(Object... paths) { + dependsOnForDeployTask(paths); + dependsOnForStandaloneDeployTask(paths); + } + + @Override + public void dependsOnForDeployTask(Object... paths) { + deployTask.configure(y -> y.dependsOn(paths)); + } + + @Override + public void dependsOnForStandaloneDeployTask(Object... paths) { + deployTask.configure(y -> y.dependsOn(paths)); + } + + @Override + public List> getPreWorkerThread() { + return preWorkerThread; + } + + @Override + public Property getDirectory() { + return directory; + } + + @Override + public List> getPredeploy() { + return predeploy; + } + + @Override + public List> getPostdeploy() { + return postdeploy; + } + + public Predicate getOnlyIf() { + return onlyIf; + } + + @Override + public void setOnlyIf(Predicate action) { + onlyIf = action; + } + + @Override + public boolean isEnabled(DeployContext context) { + if (disabled) return false; + if (onlyIf == null) return true; + if (onlyIf.test(context)) return true; + if (context != null) { + DeployLocation loc = context.getDeployLocation(); + if (loc != null) { + RemoteTarget target = loc.getTarget(); + if (target != null) { + return target.isDry(); + } + } + } + return false; + } + + @Override + public boolean isDisabled() { + return disabled; + } + + @Override + public void setDisabled() { + setDisabled(true); + } + + public void setDisabled(boolean state) { + this.disabled = state; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + "[" + this.name + "]"; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ActionArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ActionArtifact.java new file mode 100644 index 0000000..395b5cc --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ActionArtifact.java @@ -0,0 +1,31 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import javax.inject.Inject; + +import org.gradle.api.Action; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class ActionArtifact extends AbstractArtifact { + private Action deployAction; + + @Inject + public ActionArtifact(String name, RemoteTarget target) { + super(name, target); + } + + @Override + public void deploy(DeployContext context) { + deployAction.execute(context); + } + + public Action getDeployAction() { + return deployAction; + } + + public void setDeployAction(Action deployAction) { + this.deployAction = deployAction; + } + +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/Artifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/Artifact.java new file mode 100644 index 0000000..11dbe47 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/Artifact.java @@ -0,0 +1,54 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.util.List; +import java.util.function.Predicate; + +import org.gradle.api.Action; +import org.gradle.api.Named; +import org.gradle.api.plugins.ExtensionAware; +import org.gradle.api.plugins.ExtensionContainer; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.TaskProvider; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public interface Artifact extends Named { + TaskProvider getDeployTask(); + + TaskProvider getStandaloneDeployTask(); + + void allowStandaloneDeploy(); + + RemoteTarget getTarget(); + + void dependsOn(Object... paths); + + void dependsOnForDeployTask(Object... paths); + + void dependsOnForStandaloneDeployTask(Object... paths); + + List> getPreWorkerThread(); + + Property getDirectory(); + + List> getPredeploy(); + + List> getPostdeploy(); + + void setOnlyIf(Predicate action); + + boolean isEnabled(DeployContext context); + + boolean isDisabled(); + void setDisabled(); + + void deploy(DeployContext context); + + public default ExtensionContainer getExtensionContainer() { + if (this instanceof ExtensionAware) { + return ((ExtensionAware)this).getExtensions(); + } + return null; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployParameters.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployParameters.java new file mode 100644 index 0000000..4026588 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployParameters.java @@ -0,0 +1,11 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import org.gradle.api.provider.Property; +import org.gradle.workers.WorkParameters; + +import edu.wpi.first.deployutils.deploy.StorageService; + +public interface ArtifactDeployParameters extends WorkParameters { + Property getStorageService(); + Property getIndex(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployTask.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployTask.java new file mode 100644 index 0000000..1be0277 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployTask.java @@ -0,0 +1,54 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import javax.inject.Inject; + +import org.gradle.api.Action; +import org.gradle.api.DefaultTask; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.TaskAction; +import org.gradle.workers.WorkerExecutor; + +import edu.wpi.first.deployutils.deploy.StorageService; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public abstract class ArtifactDeployTask extends DefaultTask { + + @Inject + public abstract WorkerExecutor getWorkerExecutor(); + + @Input + public abstract Property getTarget(); + @Input + public abstract Property getArtifact(); + + @Internal + public abstract Property getStorageService(); + + @TaskAction + public void deployArtifact() { + Logger log = Logging.getLogger(toString()); + Artifact artifact = getArtifact().get(); + RemoteTarget target = getTarget().get(); + StorageService storageService = getStorageService().get(); + + log.debug("Deploying artifact " + artifact.getName() + " for target " + target.getName()); + + for (Action toExecute : artifact.getPreWorkerThread()) { + toExecute.execute(artifact); + } + + DeployContext ctx = target.getTargetDiscoveryTask().get().getActiveContext(); + int index = storageService.submitDeployStorage(ctx, artifact); + getWorkerExecutor().noIsolation().submit(ArtifactDeployWorker.class, config -> { + config.getStorageService().set(getStorageService()); + config.getIndex().set(index); + }); + log.debug("Workers submitted..."); + } + +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployWorker.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployWorker.java new file mode 100644 index 0000000..20fe6c9 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactDeployWorker.java @@ -0,0 +1,29 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import org.gradle.workers.WorkAction; + +import edu.wpi.first.deployutils.deploy.StorageService.DeployStorage; +import edu.wpi.first.deployutils.deploy.context.DeployContext; + +public abstract class ArtifactDeployWorker implements WorkAction { + + @Override + public void execute() { + Integer index = getParameters().getIndex().get(); + DeployStorage storage = getParameters().getStorageService().get().getDeployStorage(index); + + DeployContext rootContext = storage.context; + Artifact artifact = storage.artifact; + run(rootContext, artifact); + } + + public void run(DeployContext rootContext, Artifact artifact) { + DeployContext context = rootContext.subContext(artifact.getDirectory().get()); + boolean enabled = artifact.isEnabled(context); + if (enabled) { + ArtifactRunner.runDeploy(artifact, context); + } else { + context.getLogger().log("Artifact skipped"); + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactRunner.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactRunner.java new file mode 100644 index 0000000..da5cf12 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/ArtifactRunner.java @@ -0,0 +1,25 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.util.List; + +import org.gradle.api.Action; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; + +public class ArtifactRunner { + public static void runDeploy(Artifact artifact, DeployContext context) { + List> predeploy = artifact.getPredeploy(); + if (predeploy != null) { + for (Action action : predeploy) { + action.execute(context); + } + } + artifact.deploy(context); + List> postdeploy = artifact.getPostdeploy(); + if (postdeploy != null) { + for (Action action : postdeploy) { + action.execute(context); + } + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/BinaryLibraryArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/BinaryLibraryArtifact.java new file mode 100644 index 0000000..d8c5aa6 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/BinaryLibraryArtifact.java @@ -0,0 +1,80 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import org.gradle.api.file.FileCollection; +import org.gradle.api.provider.Property; +import org.gradle.nativeplatform.NativeBinarySpec; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.log.ETLogger; + +import java.io.File; +import java.util.Optional; +import java.util.Set; + +import javax.inject.Inject; + +public class BinaryLibraryArtifact extends AbstractArtifact implements CacheableArtifact { + private Set files; + private boolean doDeploy = false; + private final Property cacheMethod; + + private NativeBinarySpec binary; + + @Inject + public BinaryLibraryArtifact(String name, RemoteTarget target) { + super(name, target); + + cacheMethod = target.getProject().getObjects().property(CacheMethod.class); + + getPreWorkerThread().add(v -> { + Optional libs = binary.getLibs().stream().map(x -> x.getRuntimeFiles()).reduce((a, b) -> a.plus(b)); + if (libs.isPresent()) { + files = libs.get().getFiles(); + doDeploy = true; + } + }); + } + + @Override + public Property getCacheMethod() { + return cacheMethod; + } + + public NativeBinarySpec getBinary() { + return binary; + } + + public void setBinary(NativeBinarySpec binary) { + this.binary = binary; + } + + public boolean isDoDeploy() { + return doDeploy; + } + + public void setDoDeploy(boolean doDeploy) { + this.doDeploy = doDeploy; + } + + public Set getFiles() { + return files; + } + + public void setFiles(Set files) { + this.files = files; + } + + @Override + public void deploy(DeployContext context) { + if (doDeploy) { + context.put(files, getCacheMethod().getOrElse(null)); + } else { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.log("No file(s) provided for " + toString()); + } + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/CacheableArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/CacheableArtifact.java new file mode 100644 index 0000000..29d322a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/CacheableArtifact.java @@ -0,0 +1,9 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import org.gradle.api.provider.Property; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; + +public interface CacheableArtifact extends Artifact { + Property getCacheMethod(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/CommandArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/CommandArtifact.java new file mode 100644 index 0000000..4c26eef --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/CommandArtifact.java @@ -0,0 +1,36 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import javax.inject.Inject; + +import edu.wpi.first.deployutils.deploy.CommandDeployResult; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class CommandArtifact extends AbstractArtifact { + + private String command = null; + private CommandDeployResult result = null; + + @Inject + public CommandArtifact(String name, RemoteTarget target) { + super(name, target); + } + + @Override + public void deploy(DeployContext context) { + this.result = context.execute(command); + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public CommandDeployResult getResult() { + return result; + } + +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileArtifact.java new file mode 100644 index 0000000..292a67b --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileArtifact.java @@ -0,0 +1,56 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.io.File; + +import javax.inject.Inject; + +import org.gradle.api.provider.Property; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.log.ETLogger; + +public class FileArtifact extends AbstractArtifact implements CacheableArtifact { + + private final Property cacheMethod; + + @Inject + public FileArtifact(String name, RemoteTarget target) { + super(name, target); + + file = target.getProject().getObjects().property(File.class); + filename = target.getProject().getObjects().property(String.class); + cacheMethod = target.getProject().getObjects().property(CacheMethod.class); + } + + private final Property file; + + public Property getFile() { + return file; + } + + private final Property filename; + + public Property getFilename() { + return filename; + } + + @Override + public Property getCacheMethod() { + return cacheMethod; + } + + @Override + public void deploy(DeployContext context) { + if (file.isPresent()) { + File f = file.get(); + context.put(f, filename.getOrElse(f.getName()), cacheMethod.getOrElse(null)); + } else { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.log("No file provided for " + toString()); + } + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileCollectionArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileCollectionArtifact.java new file mode 100644 index 0000000..c894826 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileCollectionArtifact.java @@ -0,0 +1,46 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import org.gradle.api.file.FileCollection; +import org.gradle.api.provider.Property; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.log.ETLogger; + +import javax.inject.Inject; + +public class FileCollectionArtifact extends AbstractArtifact implements CacheableArtifact { + + private final Property cacheMethod; + + @Inject + public FileCollectionArtifact(String name, RemoteTarget target) { + super(name, target); + files = target.getProject().getObjects().property(FileCollection.class); + cacheMethod = target.getProject().getObjects().property(CacheMethod.class); + } + + private final Property files; + + public Property getFiles() { + return files; + } + + @Override + public Property getCacheMethod() { + return cacheMethod; + } + + @Override + public void deploy(DeployContext context) { + if (files.isPresent()) + context.put(files.get().getFiles(), cacheMethod.getOrElse(null)); + else { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.log("No file(s) provided for " + toString()); + } + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileTreeArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileTreeArtifact.java new file mode 100644 index 0000000..8a4e051 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/FileTreeArtifact.java @@ -0,0 +1,64 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + +import org.gradle.api.file.FileTree; +import org.gradle.api.provider.Property; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.log.ETLogger; + +public class FileTreeArtifact extends AbstractArtifact implements CacheableArtifact { + + private final Property cacheMethod; + + @Inject + public FileTreeArtifact(String name, RemoteTarget target) { + super(name, target); + files = target.getProject().getObjects().property(FileTree.class); + cacheMethod = target.getProject().getObjects().property(CacheMethod.class); + } + + private final Property files; + + public Property getFiles() { + return files; + } + + @Override + public Property getCacheMethod() { + return cacheMethod; + } + + @Override + public void deploy(DeployContext context) { + if (files.isPresent()) { + Map f = new HashMap<>(); + Set mkdirs = new HashSet<>(); + // TODO: we can probably use filevisit in dep root finding. + files.get().visit(details -> { + if (details.isDirectory()) { + mkdirs.add(details.getPath()); + } else { + f.put(details.getPath(), details.getFile()); + } + }); + + context.execute("mkdir -p " + String.join(" ", mkdirs)); + context.put(f, cacheMethod.getOrElse(null)); + } else { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.log("No file tree provided for " + toString()); + } + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/JavaArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/JavaArtifact.java new file mode 100644 index 0000000..2c5a5be --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/JavaArtifact.java @@ -0,0 +1,39 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import javax.inject.Inject; + +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.bundling.Jar; + +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class JavaArtifact extends FileArtifact { + + @Inject + public JavaArtifact(String name, RemoteTarget target) { + super(name, target); + + jarProvider = target.getProject().getObjects().property(Jar.class); + + dependsOn(jarProvider); + } + + private final Property jarProvider; + + public Provider getJarProvider() { + return jarProvider; + } + + public void setJarTask(TaskProvider jarTask) { + jarProvider.set(jarTask); + getFile().set(jarTask.get().getArchiveFile().map(x -> x.getAsFile())); + } + + public void setJarTask(Jar jarTask) { + jarProvider.set(jarTask); + dependsOn(jarTask); + getFile().set(jarTask.getArchiveFile().map(x -> x.getAsFile())); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/MavenArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/MavenArtifact.java new file mode 100644 index 0000000..97b56c3 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/MavenArtifact.java @@ -0,0 +1,38 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import javax.inject.Inject; + +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.provider.Property; + +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class MavenArtifact extends FileArtifact { + + private final Property dependency; + private final Property configuration; + + @Inject + public MavenArtifact(String name, RemoteTarget target) { + super(name, target); + + dependency = target.getProject().getObjects().property(Dependency.class); + configuration = target.getProject().getObjects().property(Configuration.class); + + getPreWorkerThread().add(cfg -> { + if (!configuration.isPresent() || !dependency.isPresent()) { + return; + } + getFile().set(configuration.get().getSingleFile()); + }); + } + + public Property getDependency() { + return dependency; + } + + public Property getConfiguration() { + return configuration; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/MultiCommandArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/MultiCommandArtifact.java new file mode 100644 index 0000000..3065dac --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/MultiCommandArtifact.java @@ -0,0 +1,50 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.inject.Inject; + +import org.gradle.api.GradleException; + +import edu.wpi.first.deployutils.deploy.CommandDeployResult; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class MultiCommandArtifact extends AbstractArtifact { + + + // Mapping name to command + // Linked hash map is ordered by insertion order + private Map commandNameMap = new LinkedHashMap<>(); + private Map resultMap = new HashMap<>(); + + @Inject + public MultiCommandArtifact(String name, RemoteTarget target) { + super(name, target); + } + + @Override + public void deploy(DeployContext context) { + for (Map.Entry command : commandNameMap.entrySet()) { + CommandDeployResult result = context.execute(command.getValue()); + resultMap.put(command.getKey(), result); + } + } + + public Map getCommands() { + return commandNameMap; + } + + public void addCommand(String name, String command) { + String oldCommand = commandNameMap.putIfAbsent(name, command); + if (oldCommand != null) { + throw new GradleException("Key already exists"); + } + } + + public Map getResults() { + return resultMap; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/NativeExecutableArtifact.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/NativeExecutableArtifact.java new file mode 100644 index 0000000..ceb40e0 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/NativeExecutableArtifact.java @@ -0,0 +1,104 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.io.File; + +import javax.inject.Inject; + +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.util.PatternFilterable; +import org.gradle.api.tasks.util.PatternSet; +import org.gradle.nativeplatform.NativeExecutableBinarySpec; +import org.gradle.nativeplatform.tasks.InstallExecutable; + +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public class NativeExecutableArtifact extends AbstractArtifact implements CacheableArtifact { + + private final Property cacheMethod; + + @Inject + public NativeExecutableArtifact(String name, RemoteTarget target) { + super(name, target); + libraryDirectory = target.getProject().getObjects().property(String.class); + filename = target.getProject().getObjects().property(String.class); + binarySpec = target.getProject().getObjects().property(NativeExecutableBinarySpec.class); + cacheMethod = target.getProject().getObjects().property(CacheMethod.class); + + installTaskProvider = target.getProject().getProviders().provider(() -> { + return (InstallExecutable)binarySpec.get().getTasks().getInstall(); + }); + + dependsOn(installTaskProvider); + } + + @Override + public Property getCacheMethod() { + return cacheMethod; + } + + private boolean deployLibraries = true; + private final Property libraryDirectory; + + private final Property filename; + + public Property getFilename() { + return filename; + } + + private Property binarySpec; + + public Property getBinary() { + return binarySpec; + } + + public boolean isDeployLibraries() { + return deployLibraries; + } + + public void setDeployLibraries(boolean deployLibraries) { + this.deployLibraries = deployLibraries; + } + + public Property getLibraryDirectory() { + return libraryDirectory; + } + + private final PatternFilterable libraryFilter = new PatternSet(); + + public PatternFilterable getLibraryFilter() { + return libraryFilter; + } + + private final Provider installTaskProvider; + + public Provider getInstallTaskProvider() { + return installTaskProvider; + } + + protected File getDeployedFile() { + InstallExecutable install = (InstallExecutable)binarySpec.get().getTasks().getInstall(); + return install.getExecutableFile().get().getAsFile(); + } + + @Override + public void deploy(DeployContext context) { + InstallExecutable install = (InstallExecutable)binarySpec.get().getTasks().getInstall(); + + CacheMethod cm = cacheMethod.getOrElse(null); + + File exeFile = getDeployedFile(); + context.put(exeFile, getFilename().getOrElse(exeFile.getName()), cm); + + if (deployLibraries) { + DeployContext libCtx = context; + if (libraryDirectory.isPresent()) { + libCtx = context.subContext(libraryDirectory.get()); + } + var libFiles = install.getLibs().getAsFileTree().matching(libraryFilter).getFiles(); + libCtx.put(libFiles, cm); + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/WrappedArrayList.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/WrappedArrayList.java new file mode 100644 index 0000000..bb9efe1 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/artifact/WrappedArrayList.java @@ -0,0 +1,31 @@ +package edu.wpi.first.deployutils.deploy.artifact; + +import java.util.ArrayList; +import java.util.Collection; + +import org.gradle.api.Action; + +import edu.wpi.first.deployutils.ActionWrapper; +import groovy.lang.Closure; + +public class WrappedArrayList extends ArrayList> { + private static final long serialVersionUID = -7867949793855347981L; + + public WrappedArrayList() { + super(); + } + + public WrappedArrayList(Collection> c) { + super(c); + } + + public WrappedArrayList(int initialCapacity) { + super(initialCapacity); + } + + public WrappedArrayList leftShift(Closure closure) { + Action wrapper = new ActionWrapper(closure); + this.add(wrapper); + return this; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/AbstractCacheMethod.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/AbstractCacheMethod.java new file mode 100644 index 0000000..356e2ab --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/AbstractCacheMethod.java @@ -0,0 +1,21 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import javax.inject.Inject; + +public abstract class AbstractCacheMethod implements CacheMethod { + private String name; + + public void setName(String name) { + this.name = name; + } + + @Inject + public AbstractCacheMethod(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CacheCheckerFunction.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CacheCheckerFunction.java new file mode 100644 index 0000000..701c19a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CacheCheckerFunction.java @@ -0,0 +1,10 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import java.io.File; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; + +@FunctionalInterface +public interface CacheCheckerFunction { + boolean check(DeployContext ctx, String filename, File localFile); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CacheMethod.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CacheMethod.java new file mode 100644 index 0000000..81d25de --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CacheMethod.java @@ -0,0 +1,15 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import java.io.File; +import java.util.Map; +import java.util.Set; + +import org.gradle.api.Named; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; + +public interface CacheMethod extends Named { + // Returns false if something can't be found (e.g. md5sum). In this case, cache checking is skipped. + boolean compatible(DeployContext context); + Set needsUpdate(DeployContext context, Map files); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CompatibleFunction.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CompatibleFunction.java new file mode 100644 index 0000000..feb257a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/CompatibleFunction.java @@ -0,0 +1,8 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; + +@FunctionalInterface +public interface CompatibleFunction { + boolean check(DeployContext ctx); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/DefaultCacheMethod.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/DefaultCacheMethod.java new file mode 100644 index 0000000..ce94749 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/DefaultCacheMethod.java @@ -0,0 +1,47 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import java.io.File; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; + +public class DefaultCacheMethod extends AbstractCacheMethod { + private CacheCheckerFunction needsUpdate = (ctx, fn, lf) -> true; + private CompatibleFunction compatible = ctx -> true; + + public DefaultCacheMethod(String name) { + super(name); + + } + + public CacheCheckerFunction getNeedsUpdate() { + return needsUpdate; + } + + public void setNeedsUpdate(CacheCheckerFunction needsUpdate) { + this.needsUpdate = needsUpdate; + } + + public CompatibleFunction getCompatible() { + return compatible; + } + + public void setCompatible(CompatibleFunction compatible) { + this.compatible = compatible; + } + + @Override + public boolean compatible(DeployContext context) { + return compatible.check(context); + } + + @Override + public Set needsUpdate(DeployContext context, Map files) { + return files.entrySet().stream() + .filter(entry -> needsUpdate.check(context, entry.getKey(), entry.getValue())) + .map(entry -> entry.getKey()) + .collect(Collectors.toSet()); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/Md5FileCacheMethod.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/Md5FileCacheMethod.java new file mode 100644 index 0000000..479db9c --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/Md5FileCacheMethod.java @@ -0,0 +1,102 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import org.codehaus.groovy.runtime.EncodingGroovyMethods; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.log.ETLogger; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import javax.inject.Inject; + +public class Md5FileCacheMethod extends AbstractCacheMethod { + private Logger log = Logging.getLogger(Md5SumCacheMethod.class); + private int csI = 0; + private Gson gson = new Gson(); + private Type mapType = new TypeToken>(){}.getType(); + + @Inject + public Md5FileCacheMethod(String name) { + super(name); + } + + @Override + public boolean compatible(DeployContext context) { + return true; + } + + private Map getRemoteCache(DeployContext ctx) { + String remote_cache = ctx.execute("cat cache.md5 2> /dev/null || echo '{}'").getResult(); + return gson.fromJson(remote_cache, mapType); + } + + public Map localChecksumsMap(Map files) { + MessageDigest md; + try { + md = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e1) { + throw new RuntimeException(e1); + } + return files.entrySet().stream().collect(Collectors.toMap(entry -> { + return entry.getKey(); + }, entry -> { + md.reset(); + try { + md.update(Files.readAllBytes(entry.getValue().toPath())); + } catch (IOException e) { + throw new RuntimeException(e); + } + return EncodingGroovyMethods.encodeHex(md.digest()).toString(); + })); + } + + @Override + public Set needsUpdate(DeployContext context, Map files) { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.silent(true); + } + int cs = csI++; + log.debug("Comparing File Checksum " + cs + "..."); + + Map remote_md5 = getRemoteCache(context); + + if (log.isDebugEnabled()) { + log.debug("Remote Cache " + cs + ":"); + log.debug(gson.toJson(remote_md5, mapType)); + } + + Map local_md5 = localChecksumsMap(files); + + if (log.isDebugEnabled()) { + log.debug("Local JSON Cache " + cs + ":"); + log.debug(gson.toJson(local_md5, mapType)); + } + + Set needs_update = files.keySet().stream().filter(name -> { + String md5 = remote_md5.get(name); + return md5 == null || !md5.equals(local_md5.get(name)); + }).collect(Collectors.toSet()); + + if (needs_update.size() > 0) { + context.execute("echo '" + gson.toJson(local_md5, mapType) + "' > cache.md5"); + } + + if (logger != null) { + logger.silent(false); + } + return needs_update; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/Md5SumCacheMethod.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/Md5SumCacheMethod.java new file mode 100644 index 0000000..3c4788a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/cache/Md5SumCacheMethod.java @@ -0,0 +1,109 @@ +package edu.wpi.first.deployutils.deploy.cache; + +import org.codehaus.groovy.runtime.EncodingGroovyMethods; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.log.ETLogger; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +public class Md5SumCacheMethod extends AbstractCacheMethod { + private Logger log = Logging.getLogger(Md5SumCacheMethod.class); + private int csI = 0; + + @Inject + public Md5SumCacheMethod(String name) { + super(name); + } + + @Override + public boolean compatible(DeployContext context) { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.silent(true); + } + String sum = context.execute("echo test | md5sum 2> /dev/null").getResult(); + if (logger != null) { + logger.silent(false); + } + + return !sum.isEmpty() && sum.split(" ")[0].equalsIgnoreCase("d8e8fca2dc0f896fd7cb4cb0031ba249"); + } + + String localChecksumsText(Map files) { + MessageDigest md; + try { + md = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e1) { + throw new RuntimeException(e1); + } + Optional sums = files.entrySet().stream().map(entry -> { + md.reset(); + try { + md.update(Files.readAllBytes(entry.getValue().toPath())); + } catch (IOException e) { + throw new RuntimeException(e); + } + String local = EncodingGroovyMethods.encodeHex(md.digest()).toString(); + return local + " *" + entry.getKey(); + }).reduce((a, b) -> a + "\n" + b); + if (sums.isEmpty()) { + return null; + } + return sums.get(); + } + + @Override + public Set needsUpdate(DeployContext context, Map files) { + ETLogger logger = context.getLogger(); + if (logger != null) { + logger.silent(true); + } + + int cs = csI++; + + log.debug("Comparing Checksums " + cs + "..."); + String localChecksums = localChecksumsText(files); + + if (log.isDebugEnabled()) { + log.debug("Local Checksums " + cs + ":"); + log.debug(localChecksums); + } + + String tmpFileName = "_tmp" + UUID.randomUUID().toString().toLowerCase().replace("-", "") + ".et.md5"; + + String result = context.execute("echo '" + localChecksums + "' > " + tmpFileName + " && md5sum -c " + tmpFileName + " 2> /dev/null; rm " + tmpFileName).getResult(); + + if (log.isDebugEnabled()) { + log.debug("Remote Checksums " + cs + ":"); + log.debug(result); + } + + List upToDate = Arrays.stream(result.split("\n")) + .map(x -> x.split(":")) + .filter(ls -> ls[ls.length - 1].trim().equalsIgnoreCase("ok")) + .map(ls -> ls[0]) + .collect(Collectors.toList()); + + if (logger != null) { + logger.silent(false); + } + + return files.keySet().stream().filter(name -> !upToDate.contains(name)).collect(Collectors.toSet()); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/context/DefaultDeployContext.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/context/DefaultDeployContext.java new file mode 100644 index 0000000..cee7346 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/context/DefaultDeployContext.java @@ -0,0 +1,128 @@ +package edu.wpi.first.deployutils.deploy.context; + +import java.io.File; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import edu.wpi.first.deployutils.PathUtils; +import edu.wpi.first.deployutils.deploy.CommandDeployResult; +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.sessions.SessionController; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; +import edu.wpi.first.deployutils.log.ETLogger; + +public class DefaultDeployContext implements DeployContext { + + private final SessionController session; + private final ETLogger logger; + private final DeployLocation deployLocation; + private final String workingDir; + + @Inject + public DefaultDeployContext(SessionController session, ETLogger logger, DeployLocation deployLocation, + String workingDir) { + this.session = session; + this.logger = logger; + this.deployLocation = deployLocation; + this.workingDir = workingDir; + } + + @Override + public String getWorkingDir() { + return workingDir; + } + + @Override + public DeployLocation getDeployLocation() { + return deployLocation; + } + + @Override + public ETLogger getLogger() { + return logger; + } + + @Override + public SessionController getController() { + return session; + } + + @Override + public CommandDeployResult execute(String command) { + session.execute("mkdir -p " + workingDir); + + logger.log(" -C-> " + command + " @ " + workingDir); + CommandDeployResult result = session.execute(String.join("\n", "cd " + workingDir, command)); + if (result != null) { + if (result.getResult() != null && result.getResult().length() > 0) { + logger.log(" -[" + result.getExitCode() + "]-> " + result.getResult()); + } else if (result.getExitCode() != 0) { + logger.log(" -[" + result.getExitCode() + "]"); + } + } + return result; + } + + @Override + public void put(Map files, CacheMethod cache) { + session.execute("mkdir -p " + workingDir); + + Map cacheHit = new HashMap<>(); + Map cacheMiss = new HashMap<>(files); + + if (cache != null && cache.compatible(this)) { + Set updateRequired = cache.needsUpdate(this, files); + for (String string : files.keySet()) { + if (updateRequired.contains(string)) continue; + cacheHit.put(string, files.get(string)); + } + for (String string : cacheHit.keySet()) { + cacheMiss.remove(string); + } + } + + if (!cacheMiss.isEmpty()) { + Map entries = cacheMiss.entrySet().stream().map(x -> { + logger.log(" -F-> " + x.getValue() + " -> " + x.getKey() + " @ " + workingDir); + return x; + }).collect(Collectors.toMap(x -> PathUtils.combine(workingDir, x.getKey()), x -> x.getValue())); + session.put(entries); + } + + if (cacheHit.size() > 0) { + logger.log(" " + cacheHit.size() + " file(s) are up-to-date and were not deployed"); + } + } + + @Override + public void put(File source, String dest, CacheMethod cache) { + put(Map.of(dest, source), cache); + } + + + + @Override + public void put(Set files, CacheMethod cache) { + put(files.stream().collect(Collectors.toMap(x -> x.getName(), x -> x)), cache); + } + + @Override + public String friendlyString() { + return session.friendlyString(); + } + + @Override + public DeployContext subContext(String workingDir) { + return new DefaultDeployContext(session, logger.push(), deployLocation, PathUtils.combine(this.workingDir, workingDir)); + } + + @Override + public void put(InputStream source, String dest) { + session.put(source, dest); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/context/DeployContext.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/context/DeployContext.java new file mode 100644 index 0000000..0ab07f9 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/context/DeployContext.java @@ -0,0 +1,40 @@ +package edu.wpi.first.deployutils.deploy.context; + +import java.io.File; +import java.io.InputStream; +import java.util.Map; +import java.util.Set; + +import edu.wpi.first.deployutils.deploy.CommandDeployResult; +import edu.wpi.first.deployutils.deploy.cache.CacheMethod; +import edu.wpi.first.deployutils.deploy.sessions.SessionController; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; +import edu.wpi.first.deployutils.log.ETLogger; + +public interface DeployContext { + SessionController getController(); + + ETLogger getLogger(); + + String getWorkingDir(); + + DeployLocation getDeployLocation(); + + CommandDeployResult execute(String command); + + // Send a batch of files + void put(Map files, CacheMethod cache); + + // Send a single file + void put(File source, String dest, CacheMethod cache); + + // Send multiple files, and trigger cache checking only once + void put(Set files, CacheMethod cache); + + // Put an input stream, with no caching + void put(InputStream source, String dest); + + String friendlyString(); + + DeployContext subContext(String workingDir); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/AbstractSessionController.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/AbstractSessionController.java new file mode 100644 index 0000000..d1e7535 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/AbstractSessionController.java @@ -0,0 +1,54 @@ +package edu.wpi.first.deployutils.deploy.sessions; + +import java.util.concurrent.Semaphore; + +import javax.inject.Inject; + +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; + +import edu.wpi.first.deployutils.deploy.StorageService; + +public abstract class AbstractSessionController implements SessionController { + private Semaphore semaphore; + private Logger log; + private int semI; + + @Inject + public AbstractSessionController(int maxConcurrent, StorageService storage) { + if (storage != null) { + storage.addSessionForCleanup(this); + } + semaphore = new Semaphore(maxConcurrent); + semI = 0; + } + + protected int acquire() { + int sem = semI++; + getLogger().debug("Acquiring Semaphore " + sem + " (" + semaphore.availablePermits() + " available)"); + long before = System.currentTimeMillis(); + try { + semaphore.acquire(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + long time = System.currentTimeMillis() - before; + getLogger().debug("Semaphore " + sem + " acquired (took " + time + "ms)"); + return sem; + } + + protected void release(int sem) { + semaphore.release(); + log.debug("Semaphore " + sem + " released"); + } + + protected Logger getLogger() { + if (log == null) log = Logging.getLogger(toString()); + return log; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + "[]"; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/DrySessionController.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/DrySessionController.java new file mode 100644 index 0000000..d5072b2 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/DrySessionController.java @@ -0,0 +1,55 @@ +package edu.wpi.first.deployutils.deploy.sessions; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import javax.inject.Inject; + +import edu.wpi.first.deployutils.deploy.CommandDeployResult; + +public class DrySessionController extends AbstractSessionController implements IPSessionController { + + @Inject + public DrySessionController() { + super(1, null); + } + + @Override + public void open() { + getLogger().info("DrySessionController opening"); + } + + @Override + public CommandDeployResult execute(String command) { + return new CommandDeployResult(command, "", 0); + } + + @Override + public void put(Map files) { } + + @Override + public String friendlyString() { + return "DrySessionController"; + } + + @Override + public void close() throws IOException { + getLogger().info("DrySessionController closing"); + } + + @Override + public String getHost() { + return "dryhost"; + } + + @Override + public int getPort() { + return 22; + } + + @Override + public void put(InputStream source, String dest) { + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/IPSessionController.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/IPSessionController.java new file mode 100644 index 0000000..498433c --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/IPSessionController.java @@ -0,0 +1,6 @@ +package edu.wpi.first.deployutils.deploy.sessions; + +public interface IPSessionController extends SessionController { + String getHost(); + int getPort(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SessionController.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SessionController.java new file mode 100644 index 0000000..ac630a9 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SessionController.java @@ -0,0 +1,19 @@ +package edu.wpi.first.deployutils.deploy.sessions; + +import java.io.File; +import java.io.InputStream; +import java.util.Map; + +import edu.wpi.first.deployutils.deploy.CommandDeployResult; + +public interface SessionController extends AutoCloseable { + void open(); + + CommandDeployResult execute(String command); + + void put(Map files); + + void put(InputStream source, String dest); + + String friendlyString(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SshSessionController.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SshSessionController.java new file mode 100644 index 0000000..c645b56 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/sessions/SshSessionController.java @@ -0,0 +1,168 @@ +package edu.wpi.first.deployutils.deploy.sessions; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; + +import org.codehaus.groovy.runtime.IOGroovyMethods; + +import edu.wpi.first.deployutils.DeployUtils; +import edu.wpi.first.deployutils.deploy.CommandDeployResult; +import edu.wpi.first.deployutils.deploy.StorageService; + +public class SshSessionController extends AbstractSessionController implements IPSessionController { + + private Session session; + private String host, user; + private int port, timeout; + + public SshSessionController(String host, int port, String user, String password, int timeout, int maxConcurrent, StorageService storage) { + super(maxConcurrent, storage); + this.host = host; + this.port = port; + this.user = user; + this.timeout = timeout; + + try { + this.session = DeployUtils.getJsch().getSession(user, host, port); + } catch (JSchException e) { + throw new RuntimeException(e); + } + this.session.setPassword(password); + + Properties config = new Properties(); + config.put("StrictHostKeyChecking", "no"); + config.put("PreferredAuthentications", "password"); + this.session.setConfig(config); + } + + @Override + public void open() { + getLogger().info("Connecting to session (timeout=" + timeout + ")"); + try { + session.setTimeout(timeout * 1000); + session.connect(timeout * 1000); + } catch (JSchException e) { + throw new RuntimeException(e); + } + + getLogger().info("Connected!"); + } + + public CommandDeployResult execute(String command) { + int sem = acquire(); + + ChannelExec exec; + try { + exec = (ChannelExec) session.openChannel("exec"); + exec.setCommand(command); + exec.setPty(false); + exec.setAgentForwarding(false); + + InputStream is = exec.getInputStream(); + exec.connect(); + exec.run(); + String result = null; + try { + result = IOGroovyMethods.getText(is); + } finally { + // Wait up to 5 seconds for closed + // isClosed must be true for getExecStatus to be correct. + long start = System.currentTimeMillis(); + while(!exec.isClosed()) { + long delta = System.currentTimeMillis() - start; + if (delta > 5000) { // 5 seconds + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + exec.disconnect(); + release(sem); + } + return new CommandDeployResult(command, result, exec.getExitStatus()); + } catch (JSchException | IOException e) { + throw new RuntimeException(e); + } + + } + + public void put(Map files) { + int sem = acquire(); + + ChannelSftp sftp; + try { + sftp = (ChannelSftp) session.openChannel("sftp"); + sftp.connect(); + try { + for (Map.Entry file : files.entrySet()) { + sftp.put(file.getValue().getAbsolutePath(), file.getKey()); + } + } finally { + sftp.disconnect(); + release(sem); + } + } catch (JSchException | SftpException e2) { + throw new RuntimeException(e2); + } + + } + + @Override + public void close() throws IOException { + try { + session.disconnect(); + } catch (Exception e) { } + } + + @Override + public String friendlyString() { + return user + "@" + host + ":" + port; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + "[" + friendlyString() + "]"; + } + + @Override + public String getHost() { + return this.host; + } + + @Override + public int getPort() { + return this.port; + } + + @Override + public void put(InputStream source, String dest) { + int sem = acquire(); + + ChannelSftp sftp; + try { + sftp = (ChannelSftp) session.openChannel("sftp"); + sftp.connect(); + try { + sftp.put(source, dest); + } finally { + sftp.disconnect(); + release(sem); + } + } catch (JSchException | SftpException e2) { + throw new RuntimeException(e2); + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/ListTypeClassesTask.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/ListTypeClassesTask.java new file mode 100644 index 0000000..79fe640 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/ListTypeClassesTask.java @@ -0,0 +1,47 @@ +package edu.wpi.first.deployutils.deploy.target; + +import java.util.Set; + +import org.gradle.api.DefaultTask; +import org.gradle.api.internal.PolymorphicDomainObjectContainerInternal; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.UntrackedTask; + +import edu.wpi.first.deployutils.deploy.artifact.Artifact; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; + +@UntrackedTask(because = "Helper task") +public class ListTypeClassesTask extends DefaultTask { + private RemoteTarget target; + + public void setTarget(RemoteTarget target) { + this.target = target; + } + + @SuppressWarnings("unchecked") + @TaskAction + public void execute() { + + getLogger().lifecycle("Type classes for {}", target.getName()); + + getLogger().lifecycle("Artifact Type Classes (getArtifactTypeClass):"); + + PolymorphicDomainObjectContainerInternal internalArtifacts = + (PolymorphicDomainObjectContainerInternal) target.getArtifacts(); + Set> artifactTypeTes = internalArtifacts.getCreateableTypes(); + for (Class artifactType : artifactTypeTes) { + getLogger().lifecycle("\t{}", artifactType.getSimpleName()); + } + + getLogger().lifecycle(""); + getLogger().lifecycle("Location Type Classes (getLocationTypeClass):"); + + PolymorphicDomainObjectContainerInternal internalLocations = + (PolymorphicDomainObjectContainerInternal) target.getLocations(); + Set> locationTypeSet = internalLocations.getCreateableTypes(); + for (Class locationType : locationTypeSet) { + getLogger().lifecycle("\t{}", locationType.getSimpleName()); + } + + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/RemoteTarget.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/RemoteTarget.java new file mode 100644 index 0000000..8615638 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/RemoteTarget.java @@ -0,0 +1,220 @@ +package edu.wpi.first.deployutils.deploy.target; + +import java.util.Set; +import java.util.function.Predicate; + +import javax.inject.Inject; + +import org.gradle.api.ExtensiblePolymorphicDomainObjectContainer; +import org.gradle.api.Named; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.internal.PolymorphicDomainObjectContainerInternal; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskProvider; + +import edu.wpi.first.deployutils.DeployUtils; +import edu.wpi.first.deployutils.deploy.DeployExtension; +import edu.wpi.first.deployutils.deploy.StorageService; +import edu.wpi.first.deployutils.deploy.artifact.Artifact; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.discovery.TargetDiscoveryTask; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; + +public class RemoteTarget implements Named { + private final Logger log; + private final String name; + private final Project project; + private final TaskProvider listTypeClassesTask; + + private final TaskProvider deployTask; + private final TaskProvider standaloneDeployTask; + private final TaskProvider targetDiscoveryTask; + private final Property targetPlatform; + private final ExtensiblePolymorphicDomainObjectContainer artifacts; + private final ExtensiblePolymorphicDomainObjectContainer locations; + private final Provider storageServiceProvider; + + public Provider getStorageServiceProvider() { + return storageServiceProvider; + } + + public ExtensiblePolymorphicDomainObjectContainer getLocations() { + return locations; + } + + public ExtensiblePolymorphicDomainObjectContainer getArtifacts() { + return artifacts; + } + + public Class getArtifactTypeClass(String name) { + @SuppressWarnings("unchecked") + PolymorphicDomainObjectContainerInternal internalArtifacts = + (PolymorphicDomainObjectContainerInternal) artifacts; + Set> artifactTypeSet = internalArtifacts.getCreateableTypes(); + for (Class artifactType : artifactTypeSet) { + if (artifactType.getSimpleName().equals(name)) { + return artifactType; + } + } + return null; + } + + public Class getLocationTypeClass(String name) { + @SuppressWarnings("unchecked") + PolymorphicDomainObjectContainerInternal internalLocations = + (PolymorphicDomainObjectContainerInternal) locations; + Set> locationTypeSet = internalLocations.getCreateableTypes(); + for (Class locationType : locationTypeSet) { + if (locationType.getSimpleName().equals(name)) { + return locationType; + } + } + return null; + } + + @Inject + public RemoteTarget(String name, Project project, DeployExtension de) { + this.name = name; + this.project = project; + this.storageServiceProvider = de.getStorageServiceProvider(); + targetPlatform = project.getObjects().property(String.class); + artifacts = project.getObjects().polymorphicDomainObjectContainer(Artifact.class); + this.dry = DeployUtils.isDryRun(project); + locations = project.getObjects().polymorphicDomainObjectContainer(DeployLocation.class); + log = Logging.getLogger(toString()); + deployTask = project.getTasks().register("deploy" + name, task -> { + task.setGroup("DeployUtils"); + task.setDescription("Deploy task for " + name); + }); + standaloneDeployTask = project.getTasks().register("deployStandalone" + name, task -> { + task.setGroup("DeployUtils"); + task.setDescription("Standalone deploy task for " + name); + }); + targetDiscoveryTask = project.getTasks().register("discover" + name, TargetDiscoveryTask.class, task -> { + task.setGroup("DeployUtils"); + task.setDescription("Determine the address(es) of target " + name); + task.setTarget(this); + task.getStorageService().set(de.getStorageServiceProvider()); + task.usesService(de.getStorageServiceProvider()); + }); + listTypeClassesTask = project.getTasks().register("listTypeClasses" + name, ListTypeClassesTask.class, task -> { + task.setGroup("DeployUtils"); + task.setDescription("Lists all type classes for a target"); + task.setTarget(this); + }); + de.getListTypeClassesTask().configure(x -> x.dependsOn(listTypeClassesTask)); + } + + public TaskProvider getListTypeClassesTask() { + return listTypeClassesTask; + } + + public Property getTargetPlatform() { + return targetPlatform; + } + + public TaskProvider getDeployTask() { + return deployTask; + } + + public TaskProvider getStandaloneDeployTask() { + return standaloneDeployTask; + } + + public TaskProvider getTargetDiscoveryTask() { + return targetDiscoveryTask; + } + + private String directory = null; + + public String getDirectory() { + return directory; + } + + public void setDirectory(String directory) { + this.directory = directory; + } + + private int timeout = 3; + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + private boolean failOnMissing = true; + + public boolean isFailOnMissing() { + return failOnMissing; + } + + public void setFailOnMissing(boolean failOnMissing) { + this.failOnMissing = failOnMissing; + } + + private int maxChannels = 1; + + public int getMaxChannels() { + return maxChannels; + } + + public void setMaxChannels(int maxChannels) { + this.maxChannels = maxChannels; + } + + private boolean dry = false; + + public boolean isDry() { + return dry; + } + + public void setDry(boolean dry) { + this.dry = dry; + } + + private Predicate onlyIf = null;; + + public Predicate getOnlyIf() { + return onlyIf; + } + + public void setOnlyIf(Predicate onlyIf) { + this.onlyIf = onlyIf; + } + + @Override + public String getName() { + return name; + } + + public Project getProject() { + return project; + } + + @Override + public String toString() { + return "RemoteTarget[" + name + "]"; + } + + public boolean verify(DeployContext ctx) { + if (onlyIf == null) { + return true; + } + + log.debug("OnlyIf..."); + boolean toConnect = onlyIf.test(ctx); + if (!toConnect) { + log.debug("OnlyIf check failed! Not connecting..."); + return false; + } + return true; + } + +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/DiscoveryFailedException.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/DiscoveryFailedException.java new file mode 100644 index 0000000..6d8b042 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/DiscoveryFailedException.java @@ -0,0 +1,18 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +import edu.wpi.first.deployutils.deploy.target.discovery.action.DiscoveryAction; + +public class DiscoveryFailedException extends Exception { + private static final long serialVersionUID = -4031180517437465326L; + + private final DiscoveryAction action; + + public DiscoveryAction getAction() { + return action; + } + + public DiscoveryFailedException(DiscoveryAction action, Throwable cause) { + super(cause); + this.action = action; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/DiscoveryState.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/DiscoveryState.java new file mode 100644 index 0000000..d290f0d --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/DiscoveryState.java @@ -0,0 +1,28 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +public enum DiscoveryState { + // STARTED and RESOLVED have the same priority since IP addresses will always pass resolution, + // but hostnames won't. So in the case no addresses can be reached, we want to sort based on + // the location order. + NOT_STARTED("not started", 0), + STARTED("failed resolution", 10), + RESOLVED("resolved but not connected", 10), + CONNECTED("connected", 20); + + private final String stateLocalized; + + public String getStateLocalized() { + return stateLocalized; + } + + private final int priority; + + public int getPriority() { + return priority; + } + + DiscoveryState(String local, int pri) { + this.stateLocalized = local; + this.priority = pri; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryTask.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryTask.java new file mode 100644 index 0000000..153b8f1 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryTask.java @@ -0,0 +1,79 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +import java.util.function.Consumer; + +import javax.inject.Inject; + +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.TaskAction; +import org.gradle.workers.WorkerExecutor; + +import edu.wpi.first.deployutils.deploy.StorageService; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.log.ETLogger; +import edu.wpi.first.deployutils.log.ETLoggerFactory; + +public abstract class TargetDiscoveryTask extends DefaultTask implements Consumer { + + @Internal + public abstract Property getStorageService(); + + @Inject + public abstract WorkerExecutor getWorkerExecutor(); + + private DeployContext activeContext; + + private RemoteTarget target; + + public void setTarget(RemoteTarget target) { + this.target = target; + } + + @Input + public RemoteTarget getTarget() { + return target; + } + + @Internal + public boolean isAvailable() { + return activeContext != null; + } + + @Internal + public DeployContext getActiveContext() { + if (activeContext != null) { + return activeContext; + } else { + throw new GradleException("Target " + target.getName() + " is not available"); + } + } + + @Override + public void accept(DeployContext ctx) { + this.activeContext = ctx; + } + + @TaskAction + public void discoverTarget() { + StorageService storageService = getStorageService().get(); + ETLogger log = ETLoggerFactory.INSTANCE.create("TargetDiscoveryTask[" + target.getName() + "]"); + + log.log("Discovering Target " + target.getName()); + int hashcode = storageService.submitDiscoveryStorage(target, this); + + // We use the Worker API since it allows for multiple of this task to run at the + // same time. Inside the worker we split off into a threadpool so we can introduce + // our own timeout logic. + log.debug("Submitting worker ${hashcode}..."); + getWorkerExecutor().noIsolation().submit(TargetDiscoveryWorker.class, config -> { + config.getStorageService().set(getStorageService()); + config.getIndex().set(hashcode); + }); + log.debug("Submitted!"); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryWorker.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryWorker.java new file mode 100644 index 0000000..7e1cb79 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryWorker.java @@ -0,0 +1,145 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +import org.gradle.workers.WorkAction; + +import edu.wpi.first.deployutils.deploy.StorageService.DiscoveryStorage; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.deploy.target.discovery.action.DiscoveryAction; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; +import edu.wpi.first.deployutils.log.ETLogger; +import edu.wpi.first.deployutils.log.ETLoggerFactory; + +public abstract class TargetDiscoveryWorker implements WorkAction { + + private ETLogger log; + + @Override + public void execute() { + DiscoveryStorage lStorage = getParameters().getStorageService().get().getDiscoveryStorage(getParameters().getIndex().get()); + Consumer callback = lStorage.contextSet; + RemoteTarget target = lStorage.target; + run(callback, target); + } + + public void run(Consumer callback, RemoteTarget target) { + log = ETLoggerFactory.INSTANCE.create(this.getClass().getSimpleName() +"[" + target.getName() + "]"); + + Collection locSet = target.getLocations(); + Set actions = new HashSet<>(locSet.size()); + for (DeployLocation deployLocation : locSet) { + actions.add(deployLocation.createAction()); + } + ExecutorService exec = Executors.newFixedThreadPool(actions.size()); + + try { + DeployContext ctx = exec.invokeAny(actions, target.getTimeout(), TimeUnit.SECONDS); + succeeded(ctx, callback, target); + } catch (TimeoutException | ExecutionException | InterruptedException ignored) { + List ex = new ArrayList<>(); + for (DiscoveryAction action : actions) { + DiscoveryFailedException e = action.getException(); + if (e == null) { + e = new DiscoveryFailedException(action, new TimeoutException("Discovery timed out.")); + } + ex.add(e); + } + failed(ex, callback, target); + } finally { + if (log.backingLogger().isInfoEnabled()) { + List ex = new ArrayList<>(); + for (DiscoveryAction action : actions) { + DiscoveryFailedException e = action.getException(); + if (e != null) { + ex.add(e); + } + } + logAllExceptions(ex); + } + } + } + + private void succeeded(DeployContext ctx, Consumer callback, RemoteTarget target) { + log.log("Using " + ctx.friendlyString() + " for target " + target.getName()); + callback.accept(ctx); + } + + private void failed(List ex, Consumer callback, RemoteTarget target) { + callback.accept(null); + log.withLock(c -> { + printFailures(ex); + String failMsg = "Target " + target.getName() + " could not be found at any location! See above for more details."; + if (target.isFailOnMissing()) { + throw new TargetNotFoundException(failMsg); + } else { + log.log(failMsg); + log.log(target.getName() + ".failOnMissing is set to false. Skipping this target and moving on..."); + } + }); + } + + private void logAllExceptions(List exceptions) { + for (DiscoveryFailedException ex : exceptions) { + log.info("Exception caught in discovery " + ex.getAction().getDeployLocation().friendlyString()); + StringWriter s = new StringWriter(); + ex.printStackTrace(new PrintWriter(s)); + log.info(s.toString()); + } + } + + private static String capitalize(String s) { + return s.substring(0, 1).toUpperCase() + s.substring(1); + } + + private void printFailures(List failures) { + Map> enumMap = new HashMap<>(); + for (DiscoveryFailedException e : failures) { + if (!enumMap.containsKey(e.getAction().getState())) { + enumMap.put(e.getAction().getState(), new ArrayList<>()); + } + enumMap.get(e.getAction().getState()).add(e); + } + + log.debug("Failures: " + enumMap); + + boolean isFirst = true; + int printFullPriority = 0; + for (DiscoveryState state : (Iterable)enumMap.keySet().stream().sorted((a,b) -> b.getPriority() - a.getPriority())::iterator) { + if (isFirst) { + isFirst = false; + printFullPriority = state.getPriority(); + } + List fails = enumMap.get(state); + if (state.getPriority() == printFullPriority || log.backingLogger().isInfoEnabled()) { + for (DiscoveryFailedException failed : fails) { + log.logErrorHead(failed.getAction().getDeployLocation().friendlyString() + ": " + capitalize(state.getStateLocalized()) + "."); + log.push().withLock(c -> { + c.logError("Reason: " + failed.getCause().getClass().getSimpleName()); + c.logError(failed.getCause().getMessage()); + }); + } + } else { + log.logErrorHead(fails.size() + " other action(s) " + state.getStateLocalized() + "."); + } + } + + log.log("Run with --info for more details"); + log.log(""); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryWorkerParameters.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryWorkerParameters.java new file mode 100644 index 0000000..64c4ad0 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetDiscoveryWorkerParameters.java @@ -0,0 +1,11 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +import org.gradle.api.provider.Property; +import org.gradle.workers.WorkParameters; + +import edu.wpi.first.deployutils.deploy.StorageService; + +public interface TargetDiscoveryWorkerParameters extends WorkParameters { + Property getStorageService(); + Property getIndex(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetNotFoundException.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetNotFoundException.java new file mode 100644 index 0000000..70fba5d --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetNotFoundException.java @@ -0,0 +1,9 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +public class TargetNotFoundException extends RuntimeException { + private static final long serialVersionUID = -4355670496129015011L; + + public TargetNotFoundException(String message) { + super(message); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetVerificationException.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetVerificationException.java new file mode 100644 index 0000000..7f01465 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/TargetVerificationException.java @@ -0,0 +1,13 @@ +package edu.wpi.first.deployutils.deploy.target.discovery; + +public class TargetVerificationException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public TargetVerificationException() { + super(); + } + + public TargetVerificationException(String msg) { + super(msg); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/AbstractDiscoveryAction.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/AbstractDiscoveryAction.java new file mode 100644 index 0000000..89aed43 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/AbstractDiscoveryAction.java @@ -0,0 +1,44 @@ +package edu.wpi.first.deployutils.deploy.target.discovery.action; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.discovery.DiscoveryFailedException; +import edu.wpi.first.deployutils.deploy.target.discovery.TargetVerificationException; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; + +public abstract class AbstractDiscoveryAction implements DiscoveryAction { + private final DeployLocation location; + private DiscoveryFailedException ex = null; + + public AbstractDiscoveryAction(DeployLocation location) { + this.location = location; + } + + @Override + public DeployLocation getDeployLocation() { + return location; + } + + @Override + public DeployContext call() throws Exception { + try { + return discover(); + } catch (Throwable t) { + DiscoveryFailedException e = new DiscoveryFailedException(this, t); + if (!(t instanceof InterruptedException)) { + ex = e; + } + throw e; + } + } + + @Override + public DiscoveryFailedException getException() { + return ex; + } + + void verify(DeployContext ctx) { + if (!location.getTarget().verify(ctx)) { + throw new TargetVerificationException("Target failed verify (onlyIf) check!"); + } + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/DiscoveryAction.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/DiscoveryAction.java new file mode 100644 index 0000000..77172bc --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/DiscoveryAction.java @@ -0,0 +1,18 @@ +package edu.wpi.first.deployutils.deploy.target.discovery.action; + +import java.util.concurrent.Callable; + +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.target.discovery.DiscoveryFailedException; +import edu.wpi.first.deployutils.deploy.target.discovery.DiscoveryState; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; + +public interface DiscoveryAction extends Callable { + DeployContext discover(); + + DiscoveryFailedException getException(); + + DiscoveryState getState(); + + DeployLocation getDeployLocation(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/DryDiscoveryAction.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/DryDiscoveryAction.java new file mode 100644 index 0000000..93a9e31 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/DryDiscoveryAction.java @@ -0,0 +1,30 @@ +package edu.wpi.first.deployutils.deploy.target.discovery.action; + +import edu.wpi.first.deployutils.deploy.context.DefaultDeployContext; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.sessions.DrySessionController; +import edu.wpi.first.deployutils.deploy.target.discovery.DiscoveryState; +import edu.wpi.first.deployutils.deploy.target.location.DeployLocation; +import edu.wpi.first.deployutils.log.ETLogger; +import edu.wpi.first.deployutils.log.ETLoggerFactory; + +public class DryDiscoveryAction extends AbstractDiscoveryAction { + + private ETLogger log; + + public DryDiscoveryAction(DeployLocation loc) { + super(loc); + this.log = ETLoggerFactory.INSTANCE.create(toString()); + } + + @Override + public DeployContext discover() { + DrySessionController controller = new DrySessionController(); + return new DefaultDeployContext(controller, log, getDeployLocation(), getDeployLocation().getTarget().getDirectory()); + } + + @Override + public DiscoveryState getState() { + return DiscoveryState.CONNECTED; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/SshDiscoveryAction.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/SshDiscoveryAction.java new file mode 100644 index 0000000..6699ab5 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/discovery/action/SshDiscoveryAction.java @@ -0,0 +1,96 @@ +package edu.wpi.first.deployutils.deploy.target.discovery.action; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +import edu.wpi.first.deployutils.deploy.context.DefaultDeployContext; +import edu.wpi.first.deployutils.deploy.context.DeployContext; +import edu.wpi.first.deployutils.deploy.sessions.SshSessionController; +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.deploy.target.discovery.DiscoveryState; +import edu.wpi.first.deployutils.deploy.target.location.SshDeployLocation; +import edu.wpi.first.deployutils.log.ETLogger; +import edu.wpi.first.deployutils.log.ETLoggerFactory; + +public class SshDiscoveryAction extends AbstractDiscoveryAction { + private DiscoveryState state = DiscoveryState.NOT_STARTED; + + private ETLogger log; + + public SshDiscoveryAction(SshDeployLocation dloc) { + super(dloc); + } + + @Override + public DiscoveryState getState() { + return state; + } + + private SshDeployLocation sshLocation() { + return (SshDeployLocation) getDeployLocation(); + } + + @Override + public DeployContext discover() { + SshDeployLocation location = sshLocation(); + RemoteTarget target = location.getTarget(); + String address = location.getAddress(); + log = ETLoggerFactory.INSTANCE.create("SshDiscoverAction[" + address + "]"); + + log.info("Discovery started..."); + state = DiscoveryState.STARTED; + + // Split host into host:port, using 22 as the default port if none provided + String[] splitHost = address.split(":"); + String hostname = splitHost[0]; + int port = splitHost.length > 1 ? Integer.parseInt(splitHost[1]) : 22; + log.info("Parsed Host: HOST = " + hostname + ", PORT = " + port); + + String resolvedHost = resolveHostname(hostname, location.isIpv6()); + state = DiscoveryState.RESOLVED; + + SshSessionController session = new SshSessionController(resolvedHost, port, location.getUser(), location.getPassword(), target.getTimeout(), location.getTarget().getMaxChannels(), getDeployLocation().getTarget().getStorageServiceProvider().get()); + session.open(); + log.info("Found " + resolvedHost + "! at " + address); + state = DiscoveryState.CONNECTED; + + DeployContext ctx = new DefaultDeployContext(session, log, location, target.getDirectory()); + log.info("Context constructed"); + + verify(ctx); + return ctx; + } + + // TODO: This should be injected to make testing easier. + private String resolveHostname(String hostname, boolean allowIpv6) { + String resolvedHost = hostname; + boolean hasResolved = false; + try { + for (InetAddress addr : InetAddress.getAllByName(hostname)) { + if (!addr.isMulticastAddress()) { + if (!allowIpv6 && addr instanceof Inet6Address) { + log.info("Resolved address " + addr.getHostAddress() + " ignored! (IPv6)"); + } else { + log.info("Resolved " + addr.getHostAddress()); + resolvedHost = addr.getHostAddress(); + hasResolved = true; + break; + } + } + } + } catch (UnknownHostException e) { + throw new RuntimeException("Unknown Host", e); + } + + if (!hasResolved) + log.info("No host resolution! Using original..."); + + return resolvedHost; + } + + @Override + public String toString() { + return this.getClass().getSimpleName(); + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/AbstractDeployLocation.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/AbstractDeployLocation.java new file mode 100644 index 0000000..945dd2a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/AbstractDeployLocation.java @@ -0,0 +1,26 @@ +package edu.wpi.first.deployutils.deploy.target.location; + +import javax.inject.Inject; + +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; + +public abstract class AbstractDeployLocation implements DeployLocation { + private final RemoteTarget target; + private final String name; + + @Inject + public AbstractDeployLocation(String name, RemoteTarget target) { + this.name = name; + this.target = target; + } + + @Override + public String getName() { + return name; + } + + @Override + public RemoteTarget getTarget() { + return this.target; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/DeployLocation.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/DeployLocation.java new file mode 100644 index 0000000..90782c5 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/DeployLocation.java @@ -0,0 +1,14 @@ +package edu.wpi.first.deployutils.deploy.target.location; + +import org.gradle.api.Named; + +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.deploy.target.discovery.action.DiscoveryAction; + +public interface DeployLocation extends Named { + DiscoveryAction createAction(); + + RemoteTarget getTarget(); + + String friendlyString(); +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/DryDeployLocation.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/DryDeployLocation.java new file mode 100644 index 0000000..d0dbc4c --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/DryDeployLocation.java @@ -0,0 +1,27 @@ +package edu.wpi.first.deployutils.deploy.target.location; + +import javax.inject.Inject; + +import edu.wpi.first.deployutils.deploy.target.discovery.action.DiscoveryAction; +import edu.wpi.first.deployutils.deploy.target.discovery.action.DryDiscoveryAction; + +public class DryDeployLocation extends AbstractDeployLocation { + + private DeployLocation inner; + + @Inject + public DryDeployLocation(String name, DeployLocation inner) { + super(name, inner.getTarget()); + this.inner = inner; + } + + @Override + public DiscoveryAction createAction() { + return new DryDiscoveryAction(inner); + } + + @Override + public String friendlyString() { + return "DryRun DeployLocation (wrapping " + inner.toString() + ")"; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/SshDeployLocation.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/SshDeployLocation.java new file mode 100644 index 0000000..d495fc0 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/deploy/target/location/SshDeployLocation.java @@ -0,0 +1,72 @@ +package edu.wpi.first.deployutils.deploy.target.location; + +import javax.inject.Inject; + +import edu.wpi.first.deployutils.deploy.target.RemoteTarget; +import edu.wpi.first.deployutils.deploy.target.discovery.action.DiscoveryAction; +import edu.wpi.first.deployutils.deploy.target.discovery.action.SshDiscoveryAction; + +public class SshDeployLocation extends AbstractDeployLocation { + private String address = null; + + private boolean ipv6 = false; + + private String user = null; + private String password = ""; + + @Inject + public SshDeployLocation(String name, RemoteTarget target) { + super(name, target); + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public boolean isIpv6() { + return ipv6; + } + + public void setIpv6(boolean ipv6) { + this.ipv6 = ipv6; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + @Override + public DiscoveryAction createAction() { + if (address == null || user == null) { + throw new IllegalArgumentException("Address and User must not be null for SshDeployLocation"); + } + return new SshDiscoveryAction(this); + } + + @Override + public String friendlyString() { + return user + " @ " + address; + } + + @Override + public String toString() { + return "SshDeployLocation[" + friendlyString() + "]"; + } + +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/log/ETLogger.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/log/ETLogger.java new file mode 100644 index 0000000..6cdfc72 --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/log/ETLogger.java @@ -0,0 +1,147 @@ +package edu.wpi.first.deployutils.log; + +import java.util.concurrent.Semaphore; +import org.gradle.api.Action; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.internal.logging.text.StyledTextOutput; + +public class ETLogger { + private int indent; + + public int getIndent() { + return indent; + } + + public void setIndent(int indent) { + this.indent = indent; + } + + private String indentStr; + + public String getIndentStr() { + return indentStr; + } + + public void setIndentStr(String indentStr) { + this.indentStr = indentStr; + } + + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + private boolean silent; + + public boolean getSilent() { + return silent; + } + + public void setSilent(boolean silent) { + this.silent = silent; + } + + private Logger internalLogger; + + public Logger getInternalLogger() { + return internalLogger; + } + + public void setInternalLogger(Logger internalLogger) { + this.internalLogger = internalLogger; + } + + private StyledTextOutput colorOut; + + public StyledTextOutput getColorOut() { + return colorOut; + } + + public void setColorOut(StyledTextOutput colorOut) { + this.colorOut = colorOut; + } + + private Semaphore semaphore; + + public ETLogger(String name, StyledTextOutput textOutput, int indent) { + this.name = name; + this.indent = indent; + this.indentStr = ""; + for (int i = 0; i < indent; i++) { + indentStr += ' '; + } + this.internalLogger = Logging.getLogger(name); + this.colorOut = textOutput; + this.semaphore = new Semaphore(1); + } + + public ETLogger(String name, StyledTextOutput textOutput) { + this(name, textOutput, 0); + } + + public ETLogger push() { + return new ETLogger(name, colorOut, indent + 2); + } + + public void withLock(Action c) { + try { + this.semaphore.acquire(); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + try { + + c.execute(this); + } finally { + this.semaphore.release(); + } + } + + public void log(String msg) { + if (!silent) System.out.println(indentStr + msg); + + if (internalLogger.isInfoEnabled()) { + internalLogger.info("Log " + (silent ? "[silent]" : "") + ": " + indentStr + msg); + } + } + + public void info(String msg) { + internalLogger.info(msg); + } + + public void debug(String msg) { + internalLogger.debug(msg); + } + + public Logger backingLogger() { + return internalLogger; + } + + public void logStyle(String msg, StyledTextOutput.Style style) { + if (colorOut != null) { + colorOut.withStyle(style).println(indentStr + msg); + } else { + log(msg); + } + } + + public void logError(String msg) { + logStyle(msg, StyledTextOutput.Style.Failure); + } + + + public void logErrorHead(String msg) { + logStyle(msg, StyledTextOutput.Style.FailureHeader); + } + + + public void silent(boolean value) { + silent = value; + } +} diff --git a/DeployUtils/src/main/java/edu/wpi/first/deployutils/log/ETLoggerFactory.java b/DeployUtils/src/main/java/edu/wpi/first/deployutils/log/ETLoggerFactory.java new file mode 100644 index 0000000..9bcc49a --- /dev/null +++ b/DeployUtils/src/main/java/edu/wpi/first/deployutils/log/ETLoggerFactory.java @@ -0,0 +1,25 @@ +package edu.wpi.first.deployutils.log; + +import org.gradle.api.Project; +import org.gradle.api.internal.project.ProjectInternal; +import org.gradle.internal.logging.text.StyledTextOutput; +import org.gradle.internal.logging.text.StyledTextOutputFactory; + +public class ETLoggerFactory { + public static ETLoggerFactory INSTANCE = new ETLoggerFactory(); + + private StyledTextOutput output = null; + + public void addColorOutput(Project project) { + StyledTextOutputFactory factory = ((ProjectInternal)project).getServices().get(StyledTextOutputFactory.class); + output = factory.create(this.getClass()); + } + + public ETLogger create(String name) { + return new ETLogger(name, output); + } + + public ETLogger create(String name, int indent) { + return new ETLogger(name, output, indent); + } +} diff --git a/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/ClosureUtilsTest.groovy b/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/ClosureUtilsTest.groovy new file mode 100644 index 0000000..6930c54 --- /dev/null +++ b/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/ClosureUtilsTest.groovy @@ -0,0 +1,34 @@ +package edu.wpi.first.deployutils + +import spock.lang.Specification + +class ClosureUtilsTest extends Specification { + + def "delegateCall delegate"() { + def delegate = Mock(DelegateSubject) + def closure = { callDelegate(it) } + + when: + ClosureUtils.delegateCall(delegate, closure) + + then: + 1 * delegate.callDelegate(delegate) + } + + def "delegateCall args"() { + def delegate = Mock(DelegateSubject) + def closure = { a,b,c -> callDelegate(b); return c } + + when: + def ret = ClosureUtils.delegateCall(delegate, closure, 12.0, 24.0) + + then: + 1 * delegate.callDelegate(12.0) + ret == 24.0 + } + + static interface DelegateSubject { + void callDelegate(Object arg) + } + +} diff --git a/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/DeployUtilsInitializationTest.groovy b/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/DeployUtilsInitializationTest.groovy new file mode 100644 index 0000000..731fee9 --- /dev/null +++ b/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/DeployUtilsInitializationTest.groovy @@ -0,0 +1,33 @@ +package edu.wpi.first.deployutils + +import org.gradle.testkit.runner.GradleRunner +import static org.gradle.testkit.runner.TaskOutcome.* +import spock.lang.TempDir +import spock.lang.Specification + +class DeployUtilsInitializationTest extends Specification { + @TempDir File testProjectDir + File buildFile + + def setup() { + buildFile = new File(testProjectDir, 'build.gradle') + } + + def "Project Initializes Correctly"() { + given: + buildFile << """plugins { + id 'cpp' + id 'edu.wpi.first.DeployUtils' +} +""" + when: + def result = GradleRunner.create() + .withProjectDir(testProjectDir) + .withArguments('tasks', '--stacktrace') + .withPluginClasspath() + .build() + + then: + result.task(':tasks').outcome == SUCCESS + } +} diff --git a/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/PathUtilsTest.groovy b/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/PathUtilsTest.groovy new file mode 100644 index 0000000..eccfd0f --- /dev/null +++ b/DeployUtils/src/test/groovy/edu/wpi/first/deployutils/PathUtilsTest.groovy @@ -0,0 +1,34 @@ +package edu.wpi.first.deployutils + +import spock.lang.Specification + +class PathUtilsTest extends Specification { + + def "combine"() { + when: + // tests mostly join + def path = PathUtils.combine("/myroot/", "relative/") + + then: + path.equals("/myroot/relative") + } + + def "combine upone"() { + when: + // tests normalize, for .. + def path = PathUtils.combine("/some/deep/directory", "../directory2") + + then: + path.equals("/some/deep/directory2") + } + + def "combine to root"() { + when: + // tests mostly join for root + def path = PathUtils.combine("/some/deep/directory", "/root") + + then: + path.equals("/root") + } + +} diff --git a/ToolchainPlugin/build.gradle b/ToolchainPlugin/build.gradle index 2fb1ad0..35b6b3d 100644 --- a/ToolchainPlugin/build.gradle +++ b/ToolchainPlugin/build.gradle @@ -17,7 +17,7 @@ repositories { dependencies { // For some utility classes. We don't actually apply DeployUtils to the FRCToolchain, // but we do in GradleRIO - api 'edu.wpi.first:DeployUtils:2024.1.0' + api project(':DeployUtils') api 'de.undercouch:gradle-download-task:4.0.1' testImplementation('org.spockframework:spock-core:2.0-M4-groovy-3.0') { diff --git a/build.gradle b/build.gradle index 55254b8..85161cf 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,11 @@ java { targetCompatibility = 17 } +buildScan { + termsOfServiceUrl = 'https://gradle.com/terms-of-service' + termsOfServiceAgree = 'yes' +} + allprojects { group = "edu.wpi.first" version = "2025.1.0" diff --git a/settings.gradle b/settings.gradle index 4a19651..3b64787 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1,7 @@ +plugins { + id 'com.gradle.enterprise' version '3.15.1' +} + +rootProject.name = 'NativeUtils' include ':ToolchainPlugin' +include ':DeployUtils' diff --git a/testing/cpp/build.gradle b/testing/cpp/build.gradle index c8df876..c52c1f5 100644 --- a/testing/cpp/build.gradle +++ b/testing/cpp/build.gradle @@ -5,6 +5,13 @@ plugins { id "edu.wpi.first.NativeUtils" version "2025.1.0" } +deploy { + targets { + myTarget(getTargetTypeClass("RemoteTarget")) { + } + } +} + nativeUtils.addWpiNativeUtils() nativeUtils.withCrossRoboRIO() nativeUtils.withCrossLinuxArm32()