diff --git a/build-project.sh b/build-project.sh index 522d8b0918..037313482d 100755 --- a/build-project.sh +++ b/build-project.sh @@ -3,5 +3,5 @@ set -e HDFS_VERSION=$1 -mvn install:install-file -pl smart-tests -Dos.arch=x86_64 +mvn install:install-file -pl smart-tests mvn clean install -Pdist,web-ui,hadoop-"${HDFS_VERSION}" -DskipTests \ No newline at end of file diff --git a/smart-action/src/main/java/org/smartdata/action/AbstractActionFactory.java b/smart-action/src/main/java/org/smartdata/action/AbstractActionFactory.java index cc13b3ceb1..0ed669e47e 100644 --- a/smart-action/src/main/java/org/smartdata/action/AbstractActionFactory.java +++ b/smart-action/src/main/java/org/smartdata/action/AbstractActionFactory.java @@ -22,9 +22,11 @@ import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * A common action factory for action providers to use. @@ -34,12 +36,9 @@ public abstract class AbstractActionFactory implements ActionFactory { private static final List> COMMON_ACTIONS = Arrays.asList( EchoAction.class, SleepAction.class, - SyncAction.class, ExecAction.class ); - protected abstract List> supportedActionClasses(); - @Override public Map> getSupportedActions() { Map> supportedActions = new HashMap<>(); @@ -51,12 +50,33 @@ public Map> getSupportedActions() { return supportedActions; } + @Override + public Set getActionMetadata() { + Set actionMetadata = new HashSet<>(); + COMMON_ACTIONS.forEach(action -> + toActionMetadata(action).ifPresent(actionMetadata::add)); + supportedActionClasses().forEach(action -> + toActionMetadata(action).ifPresent(actionMetadata::add)); + + return actionMetadata; + } + + protected abstract List> supportedActionClasses(); + + private Optional toActionMetadata(Class actionClass) { + return actionSignature(actionClass) + .map(signature -> new ActionMetadata(signature.actionId(), signature.usage())); + } + private void addActionInfo( Map> supportedActions, Class actionClass) { - - Optional.ofNullable(actionClass.getAnnotation(ActionSignature.class)) + actionSignature(actionClass) .map(ActionSignature::actionId) .ifPresent(actionId -> supportedActions.put(actionId, actionClass)); } + + private Optional actionSignature(Class actionClass) { + return Optional.ofNullable(actionClass.getAnnotation(ActionSignature.class)); + } } diff --git a/smart-action/src/main/java/org/smartdata/action/ActionFactory.java b/smart-action/src/main/java/org/smartdata/action/ActionFactory.java index a66fafc263..0d5e95a0bf 100644 --- a/smart-action/src/main/java/org/smartdata/action/ActionFactory.java +++ b/smart-action/src/main/java/org/smartdata/action/ActionFactory.java @@ -18,6 +18,7 @@ package org.smartdata.action; import java.util.Map; +import java.util.Set; /** * Action factory interface. Either built-in or user defined actions will be @@ -30,4 +31,6 @@ public interface ActionFactory { * @return supported actions */ Map> getSupportedActions(); + + Set getActionMetadata(); } diff --git a/smart-action/src/main/java/org/smartdata/action/ActionMetadata.java b/smart-action/src/main/java/org/smartdata/action/ActionMetadata.java new file mode 100644 index 0000000000..3661f6d987 --- /dev/null +++ b/smart-action/src/main/java/org/smartdata/action/ActionMetadata.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.action; + +import lombok.Data; + +@Data +public class ActionMetadata { + private final String name; + private final String usage; +} diff --git a/smart-action/src/main/java/org/smartdata/action/ActionRegistry.java b/smart-action/src/main/java/org/smartdata/action/ActionRegistry.java index e75cebf09e..5b1d7a32ce 100644 --- a/smart-action/src/main/java/org/smartdata/action/ActionRegistry.java +++ b/smart-action/src/main/java/org/smartdata/action/ActionRegistry.java @@ -17,10 +17,12 @@ */ package org.smartdata.action; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -29,12 +31,21 @@ */ @Slf4j public class ActionRegistry { - private final Map> actions = new HashMap<>(); + @Getter + private final Set actionMetadata; + private final Map> actions; public ActionRegistry(Collection factories) { + this.actions = new HashMap<>(); + this.actionMetadata = new HashSet<>(); + factories.stream() .map(ActionFactory::getSupportedActions) .forEach(actions::putAll); + + factories.stream() + .map(ActionFactory::getActionMetadata) + .forEach(actionMetadata::addAll); } public Set registeredActions() { diff --git a/smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java b/smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java index c4bfede746..b7257e17ec 100644 --- a/smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java +++ b/smart-common/src/main/java/org/smartdata/metrics/GeneralFileInfoSource.java @@ -17,10 +17,17 @@ */ package org.smartdata.metrics; +import org.smartdata.model.BaseFileInfo; + import java.sql.SQLException; import java.util.Collection; +import java.util.List; import java.util.Map; public interface GeneralFileInfoSource { Map getPathsToIdsMapping(Collection paths) throws SQLException; + + List getFilePathsByPrefix(String path); + + BaseFileInfo getBaseFileInfo(String path); } diff --git a/smart-common/src/main/java/org/smartdata/model/BackUpInfo.java b/smart-common/src/main/java/org/smartdata/model/BackUpInfo.java index 300b4ddfaf..96254c964c 100644 --- a/smart-common/src/main/java/org/smartdata/model/BackUpInfo.java +++ b/smart-common/src/main/java/org/smartdata/model/BackUpInfo.java @@ -17,101 +17,23 @@ */ package org.smartdata.model; -import java.util.Objects; +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; import static org.smartdata.utils.StringUtil.ssmPatternToRegex; +@Data +@RequiredArgsConstructor +@Builder(toBuilder = true) public class BackUpInfo { - private long rid; - private String src; - private String dest; - private long period; // in milliseconds - private String srcPattern; + private final long rid; + private final String src; + private final String dest; + private final long period; // in milliseconds + private final String srcPattern; public BackUpInfo(long rid, String src, String dest, long period) { this(rid, src, dest, period, ssmPatternToRegex(src + "*")); } - - public BackUpInfo(long rid, String src, String dest, long period, String srcPattern) { - this.rid = rid; - this.src = src; - this.dest = dest; - this.period = period; - this.srcPattern = srcPattern; - } - - public BackUpInfo() { - } - - public long getRid() { - return rid; - } - - public void setRid(long rid) { - this.rid = rid; - } - - public String getSrc() { - return src; - } - - public void setSrc(String src) { - this.src = src; - } - - public String getDest() { - return dest; - } - - public void setDest(String dest) { - this.dest = dest; - } - - public long getPeriod() { - return period; - } - - public void setPeriod(long period) { - this.period = period; - } - - public String getSrcPattern() { - return srcPattern; - } - - public void setSrcPattern(String srcPattern) { - this.srcPattern = srcPattern; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - BackUpInfo that = (BackUpInfo) o; - return rid == that.rid - && period == that.period - && Objects.equals(src, that.src) - && Objects.equals(dest, that.dest) - && Objects.equals(srcPattern, that.srcPattern); - } - - @Override - public int hashCode() { - return Objects.hash(rid, src, dest, period, srcPattern); - } - - @Override - public String toString() { - return "BackUpInfo{" - + "rid=" + rid - + ", src='" + src + '\'' - + ", dest='" + dest + '\'' - + ", period=" + period - + ", srcPattern='" + srcPattern + '\'' - + '}'; - } } diff --git a/smart-common/src/main/java/org/smartdata/model/BaseFileInfo.java b/smart-common/src/main/java/org/smartdata/model/BaseFileInfo.java new file mode 100644 index 0000000000..25607c8dd2 --- /dev/null +++ b/smart-common/src/main/java/org/smartdata/model/BaseFileInfo.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.model; + +public interface BaseFileInfo { + String getPath(); + + long getLength(); + + boolean isDir(); +} diff --git a/smart-common/src/main/java/org/smartdata/model/FileInfo.java b/smart-common/src/main/java/org/smartdata/model/FileInfo.java index 516b2c4ce3..a7c4bfab7e 100644 --- a/smart-common/src/main/java/org/smartdata/model/FileInfo.java +++ b/smart-common/src/main/java/org/smartdata/model/FileInfo.java @@ -24,7 +24,7 @@ @Data @AllArgsConstructor @Builder(setterPrefix = "set") -public class FileInfo { +public class FileInfo implements BaseFileInfo { private String path; private long fileId; private long length; diff --git a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java index b10fa275df..5dfff1da24 100644 --- a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java +++ b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java @@ -65,6 +65,8 @@ public class SmartEngine extends AbstractService { private SmartPrincipalManager smartPrincipalManager; @Getter private CachedFilesManager cachedFilesManager; + @Getter + private ActionRegistry actionRegistry; private final List services; @@ -85,7 +87,7 @@ public void init() throws IOException { FileSystemContext fsContext = FileSystemContext.fromConfig(conf); - ActionRegistry actionRegistry = new ActionRegistry(fsContext.actionFactories()); + actionRegistry = new ActionRegistry(fsContext.actionFactories()); cmdletManager = CmdletManager.builder() .context(serverContext) @@ -103,6 +105,7 @@ public void init() throws IOException { .actionRegistry(actionRegistry) .smartPrincipalManager(smartPrincipalManager) .executorPlugins(fsContext.ruleExecutorPlugins(serverContext, cmdletManager)) + .smartObjectSupplier(fsContext.smartObjectSupplier()) .build(); services.add(ruleManager); diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/RuleManager.java b/smart-engine/src/main/java/org/smartdata/server/engine/RuleManager.java index b63baad789..58a10f3a96 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/RuleManager.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/RuleManager.java @@ -41,6 +41,7 @@ import org.smartdata.model.rule.RulePluginManager; import org.smartdata.model.rule.RuleTranslationResult; import org.smartdata.model.rule.TimeBasedScheduleInfo; +import org.smartdata.rule.objects.SmartObjectSupplier; import org.smartdata.rule.parser.SmartRuleStringParser; import org.smartdata.security.SmartPrincipalManager; import org.smartdata.server.engine.audit.AuditService; @@ -86,6 +87,7 @@ public class RuleManager private final RuleDao ruleDao; private final RuleInfoHandler ruleInfoHandler; private final List executorPlugins; + private final SmartObjectSupplier smartObjectSupplier; private volatile boolean isClosed = false; @@ -100,6 +102,7 @@ public RuleManager( AuditService auditService, ActionRegistry actionRegistry, SmartPrincipalManager smartPrincipalManager, + SmartObjectSupplier smartObjectSupplier, List executorPlugins) { super(context); @@ -121,6 +124,7 @@ public RuleManager( this.pathChecker = new PathChecker(context.getConf()); this.executorPlugins = executorPlugins; this.actionRegistry = actionRegistry; + this.smartObjectSupplier = smartObjectSupplier; } public RuleInfo submitRule(String rule) throws IOException { @@ -166,7 +170,8 @@ public long submitRule(String rule, RuleState initState) throws IOException { metaStore.insertNewRule(ruleInfo); - RuleInfoRepo infoRepo = new RuleInfoRepo(ruleInfo, metaStore, serverContext.getConf(), executorPlugins); + RuleInfoRepo infoRepo = new RuleInfoRepo(ruleInfo, metaStore, + serverContext.getConf(), smartObjectSupplier, executorPlugins); mapRules.put(ruleInfo.getId(), infoRepo); submitRuleToScheduler(infoRepo.launchExecutor(this)); @@ -188,7 +193,8 @@ private void doCheckActions(CmdletDescriptor cd) throws IOException { } private RuleTranslationResult doCheckRule(String rule) throws IOException { - SmartRuleStringParser parser = new SmartRuleStringParser(rule, null, serverContext.getConf()); + SmartRuleStringParser parser = new SmartRuleStringParser( + rule, null, smartObjectSupplier, serverContext.getConf()); return parser.translate(); } @@ -294,7 +300,8 @@ public void init() throws IOException { return; } for (RuleInfo rule : rules) { - mapRules.put(rule.getId(), new RuleInfoRepo(rule, metaStore, serverContext.getConf(), executorPlugins)); + mapRules.put(rule.getId(), new RuleInfoRepo(rule, metaStore, + serverContext.getConf(), smartObjectSupplier, executorPlugins)); } LOG.info("Initialized. Totally " + rules.size() + " rules loaded from DataBase."); if (LOG.isDebugEnabled()) { diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/BaseFileSystemContext.java b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/BaseFileSystemContext.java index 532498b47d..afda24f60c 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/BaseFileSystemContext.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/BaseFileSystemContext.java @@ -23,9 +23,25 @@ import org.smartdata.server.engine.ServerContext; import org.smartdata.utils.ThrowingBiFunction; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + @Slf4j public abstract class BaseFileSystemContext implements FileSystemContext { + @Override + public List actionSchedulerServices(ServerContext context) { + return actionSchedulerSuppliers() + .map(supplier -> createSafely(supplier, context)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + protected abstract Stream> actionSchedulerSuppliers(); + protected ActionSchedulerService createSafely( ThrowingBiFunction schedulerSupplier, ServerContext smartContext) { diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/FileSystemContext.java b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/FileSystemContext.java index 9fa840281f..8ed3450c07 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/FileSystemContext.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/FileSystemContext.java @@ -23,6 +23,7 @@ import org.smartdata.conf.SmartFsType; import org.smartdata.model.action.ActionSchedulerService; import org.smartdata.model.rule.RuleExecutorPlugin; +import org.smartdata.rule.objects.SmartObjectSupplier; import org.smartdata.server.engine.CmdletManager; import org.smartdata.server.engine.ServerContext; import org.smartdata.server.engine.file.CachedFilesManager; @@ -42,6 +43,8 @@ List ruleExecutorPlugins( CachedFilesManager cachedFilesManager(ServerContext context); + SmartObjectSupplier smartObjectSupplier(); + static FileSystemContext fromConfig(SmartConf conf) { return fromFsType(conf.getFsType()); } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/HdfsContext.java b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/HdfsContext.java index 882fb1407f..b17b3efd8c 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/HdfsContext.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/HdfsContext.java @@ -34,6 +34,8 @@ import org.smartdata.metastore.MetaStore; import org.smartdata.model.action.ActionSchedulerService; import org.smartdata.model.rule.RuleExecutorPlugin; +import org.smartdata.rule.objects.DefaultSmartObjectSupplier; +import org.smartdata.rule.objects.SmartObjectSupplier; import org.smartdata.server.engine.CmdletManager; import org.smartdata.server.engine.ServerContext; import org.smartdata.server.engine.file.CachedFilesManager; @@ -48,30 +50,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import java.util.stream.Stream; public class HdfsContext extends BaseFileSystemContext { - @Override - public List actionSchedulerServices(ServerContext context) { - return Stream.>of( - (ctx, metastore) -> new MoverScheduler(ctx), - CopyScheduler::new, - Copy2S3Scheduler::new, - SmallFileScheduler::new, - CompressionScheduler::new, - ErasureCodingScheduler::new, - Copy2S3Scheduler::new, - (ctx, metastore) -> new CacheScheduler(ctx), - (ctx, metastore) -> new HmsSyncScheduler(ctx, - metastore.hmsEventDao(), metastore.hmsSyncProgressDao()) - ).map(supplier -> createSafely(supplier, context)) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - @Override public List ruleExecutorPlugins(ServerContext context, CmdletManager cmdletManager) { return Arrays.asList( @@ -103,4 +85,24 @@ public List additionalServices(ServerContext context) { public CachedFilesManager cachedFilesManager(ServerContext context) { return new DbCachedFilesManager(context.getMetaStore().cacheFileDao()); } + + @Override + public SmartObjectSupplier smartObjectSupplier() { + return new DefaultSmartObjectSupplier(); + } + + @Override + protected Stream> actionSchedulerSuppliers() { + return Stream.of( + (ctx, metastore) -> new MoverScheduler(ctx), + CopyScheduler::new, + Copy2S3Scheduler::new, + SmallFileScheduler::new, + CompressionScheduler::new, + ErasureCodingScheduler::new, + (ctx, metastore) -> new CacheScheduler(ctx), + (ctx, metastore) -> new HmsSyncScheduler(ctx, + metastore.hmsEventDao(), metastore.hmsSyncProgressDao())); + } } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/OzoneFileSystemContext.java b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/OzoneFileSystemContext.java index a94359b7a2..ccccffa8b0 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/OzoneFileSystemContext.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/filesystem/OzoneFileSystemContext.java @@ -19,33 +19,47 @@ import org.smartdata.SmartService; import org.smartdata.action.ActionFactory; +import org.smartdata.hdfs.scheduler.Copy2S3Scheduler; +import org.smartdata.hdfs.scheduler.CopyScheduler; import org.smartdata.hive.action.HiveActionFactory; +import org.smartdata.hive.action.HmsSyncScheduler; +import org.smartdata.hive.rule.HmsSyncRulePlugin; +import org.smartdata.metastore.MetaStore; import org.smartdata.model.action.ActionSchedulerService; import org.smartdata.model.rule.RuleExecutorPlugin; import org.smartdata.ozone.OzoneFetcherService; +import org.smartdata.ozone.action.OzoneActionFactory; +import org.smartdata.ozone.rule.OzoneSmartObjectSupplier; +import org.smartdata.rule.objects.SmartObjectSupplier; import org.smartdata.server.engine.CmdletManager; import org.smartdata.server.engine.ServerContext; import org.smartdata.server.engine.file.CachedFilesManager; import org.smartdata.server.engine.file.NoOpCachedFilesManager; +import org.smartdata.server.engine.rule.FileCopy2S3Plugin; +import org.smartdata.server.engine.rule.copy.FileCopyDrPlugin; +import org.smartdata.server.engine.rule.copy.FileCopyScheduleStrategy; +import org.smartdata.utils.ThrowingBiFunction; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Stream; public class OzoneFileSystemContext extends BaseFileSystemContext { - @Override - public List actionSchedulerServices(ServerContext context) { - return Collections.emptyList(); - } - @Override public List ruleExecutorPlugins(ServerContext context, CmdletManager cmdletManager) { - return Collections.emptyList(); + return Arrays.asList( + new FileCopyDrPlugin( + context.getMetaStore(), FileCopyScheduleStrategy.ordered()), + new FileCopy2S3Plugin(), + new HmsSyncRulePlugin(context.getMetaStore().hmsSyncProgressDao()) + ); } @Override public List actionFactories() { return Arrays.asList( + new OzoneActionFactory(), new HiveActionFactory() ); } @@ -64,4 +78,19 @@ public List additionalServices(ServerContext context) { public CachedFilesManager cachedFilesManager(ServerContext context) { return new NoOpCachedFilesManager(); } + + @Override + public SmartObjectSupplier smartObjectSupplier() { + return new OzoneSmartObjectSupplier(); + } + + @Override + protected Stream> actionSchedulerSuppliers() { + return Stream.of( + CopyScheduler::new, + Copy2S3Scheduler::new, + (ctx, metastore) -> new HmsSyncScheduler(ctx, + metastore.hmsEventDao(), metastore.hmsSyncProgressDao())); + } } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java b/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java index 1e58b6177e..29e8d813fd 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/rule/RuleInfoRepo.java @@ -26,6 +26,7 @@ import org.smartdata.model.RuleState; import org.smartdata.model.rule.RuleExecutorPlugin; import org.smartdata.model.rule.RuleTranslationResult; +import org.smartdata.rule.objects.SmartObjectSupplier; import org.smartdata.rule.parser.SmartRuleStringParser; import org.smartdata.rule.parser.TranslationContext; import org.smartdata.server.engine.RuleManager; @@ -44,6 +45,8 @@ public class RuleInfoRepo { private final RuleDao ruleDao; private final SmartConf conf; private final List executorPlugins; + private final SmartObjectSupplier smartObjectSupplier; + private RuleExecutor ruleExecutor; private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @@ -51,10 +54,12 @@ public class RuleInfoRepo { public RuleInfoRepo(RuleInfo ruleInfo, MetaStore metaStore, SmartConf conf, + SmartObjectSupplier smartObjectSupplier, List executorPlugins) { this.ruleInfo = ruleInfo; this.metaStore = metaStore; this.ruleDao = metaStore.ruleDao(); + this.smartObjectSupplier = smartObjectSupplier; this.executorPlugins = executorPlugins; this.conf = conf; } @@ -149,7 +154,8 @@ private RuleExecutor doLaunchExecutor(RuleManager ruleManager) ruleInfo.getId(), ruleInfo.getSubmitTime()); RuleTranslationResult translationResult = ruleExecutor != null ? ruleExecutor.getOriginalTranslateResult() - : new SmartRuleStringParser(ruleInfo.getRuleText(), translationCtx, conf).translate(); + : new SmartRuleStringParser(ruleInfo.getRuleText(), + translationCtx, smartObjectSupplier, conf).translate(); ruleExecutor = new RuleExecutor( conf, diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/rule/copy/FileCopyDrPlugin.java b/smart-engine/src/main/java/org/smartdata/server/engine/rule/copy/FileCopyDrPlugin.java index 938cca78a7..1950568172 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/rule/copy/FileCopyDrPlugin.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/rule/copy/FileCopyDrPlugin.java @@ -152,14 +152,13 @@ private BackUpInfo buildBackupInfo( PATTERN_BASE_DIRS_DELIMITER, getPathPatternBaseDirs(pathPatterns)); - BackUpInfo backUpInfo = new BackUpInfo(); - backUpInfo.setRid(ruleId); - backUpInfo.setSrc(patternBaseDirs); - backUpInfo.setSrcPattern(ssmPatternsToRegex(pathPatterns)); - backUpInfo.setDest(dest); - backUpInfo.setPeriod(tResult.getScheduleInfo().getMinimalEvery()); - - return backUpInfo; + return BackUpInfo.builder() + .rid(ruleId) + .src(patternBaseDirs) + .srcPattern(ssmPatternsToRegex(pathPatterns)) + .dest(dest) + .period(tResult.getScheduleInfo().getMinimalEvery()) + .build(); } private void validatePreserveArg(String rawPreserveArg) { @@ -167,7 +166,7 @@ private void validatePreserveArg(String rawPreserveArg) { return; } - for (String attribute: rawPreserveArg.split(",")) { + for (String attribute : rawPreserveArg.split(",")) { CopyFileAction.validatePreserveArg(attribute); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java index b0e924fc71..75b60bc414 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java @@ -19,6 +19,7 @@ import org.smartdata.action.AbstractActionFactory; import org.smartdata.action.SmartAction; +import org.smartdata.action.SyncAction; import java.util.Arrays; import java.util.List; @@ -68,7 +69,8 @@ protected List> supportedActionClasses() { AddErasureCodingPolicy.class, RemoveErasureCodingPolicy.class, EnableErasureCodingPolicy.class, - DisableErasureCodingPolicy.class + DisableErasureCodingPolicy.class, + SyncAction.class ); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/ChecksumFileEqualityStrategy.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/ChecksumFileEqualityStrategy.java index 141ae654e8..befd23ae93 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/ChecksumFileEqualityStrategy.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/ChecksumFileEqualityStrategy.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.hdfs.HadoopUtil; +import org.smartdata.model.BaseFileInfo; import org.smartdata.model.FileInfo; import java.io.IOException; @@ -52,7 +53,7 @@ private FileSystem getFileSystem(Path path, Configuration conf) throws IOExcepti } @Override - public boolean areEqual(FileInfo srcFileInfo, FileStatus destFileStatus) { + public boolean areEqual(BaseFileInfo srcFileInfo, FileStatus destFileStatus) { if (!filesLengthComparator.areEqual(srcFileInfo, destFileStatus)) { // we don't need to fetch and compare checksums // if the files are obviously not equal. diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/FileEqualityStrategy.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/FileEqualityStrategy.java index b594ad788d..7755288db2 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/FileEqualityStrategy.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/FileEqualityStrategy.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.smartdata.conf.SmartConfKeys; +import org.smartdata.model.BaseFileInfo; import org.smartdata.model.FileInfo; public interface FileEqualityStrategy { @@ -29,7 +30,7 @@ enum Strategy { CHECKSUM } - boolean areEqual(FileInfo srcFileInfo, FileStatus destFileStatus); + boolean areEqual(BaseFileInfo srcFileInfo, FileStatus destFileStatus); static FileEqualityStrategy from(Configuration conf) { String rawStrategy = conf.get( diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/LengthFileEqualityStrategy.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/LengthFileEqualityStrategy.java index 9942847bde..2469f09a50 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/LengthFileEqualityStrategy.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/file/equality/LengthFileEqualityStrategy.java @@ -17,14 +17,15 @@ */ package org.smartdata.hdfs.file.equality; -import java.util.Optional; import org.apache.hadoop.fs.FileStatus; -import org.smartdata.model.FileInfo; +import org.smartdata.model.BaseFileInfo; + +import java.util.Optional; public class LengthFileEqualityStrategy implements FileEqualityStrategy { @Override - public boolean areEqual(FileInfo srcFileInfo, FileStatus destFileStatus) { + public boolean areEqual(BaseFileInfo srcFileInfo, FileStatus destFileStatus) { return Optional.ofNullable(destFileStatus) .map(FileStatus::getLen) .filter(length -> length == srcFileInfo.getLength()) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/Copy2S3Scheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/Copy2S3Scheduler.java index 4f71f9882b..71032efdd1 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/Copy2S3Scheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/Copy2S3Scheduler.java @@ -25,6 +25,7 @@ import org.smartdata.metastore.MetaStore; import org.smartdata.metastore.MetaStoreException; import org.smartdata.model.ActionInfo; +import org.smartdata.model.BaseFileInfo; import org.smartdata.model.CmdletInfo; import org.smartdata.model.FileInfo; import org.smartdata.model.FileState; @@ -123,8 +124,8 @@ private boolean isLocked(String filePath) { private Optional getFileLength(String fileName) { try { - return Optional.ofNullable(metaStore.getFile(fileName)) - .map(FileInfo::getLength); + return Optional.ofNullable(metaStore.getBaseFileInfo(fileName)) + .map(BaseFileInfo::getLength); } catch (MetaStoreException e) { LOG.warn("Error fetching info about file: {}", fileName, e); return Optional.empty(); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/CopyScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/CopyScheduler.java index f102400526..e9b05ea50c 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/CopyScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/CopyScheduler.java @@ -36,6 +36,7 @@ import org.smartdata.metastore.MetaStore; import org.smartdata.metastore.MetaStoreException; import org.smartdata.model.ActionInfo; +import org.smartdata.model.BaseFileInfo; import org.smartdata.model.CmdletInfo; import org.smartdata.model.CompressionFileState; import org.smartdata.model.FileDiff; @@ -44,16 +45,15 @@ import org.smartdata.model.FileInfo; import org.smartdata.model.FileState; import org.smartdata.model.LaunchAction; +import org.smartdata.model.action.ActionSchedulerService; import org.smartdata.model.action.ScheduleResult; import org.smartdata.protocol.message.LaunchCmdlet; -import org.smartdata.model.action.ActionSchedulerService; import java.io.IOException; import java.net.URI; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -91,7 +91,6 @@ import static org.smartdata.model.FileDiffState.PENDING; import static org.smartdata.model.FileDiffState.isTerminalState; import static org.smartdata.model.FileDiffType.DELETE; -import static org.smartdata.utils.ConfigUtil.toRemoteClusterConfig; import static org.smartdata.utils.FileDiffUtils.getDest; import static org.smartdata.utils.FileDiffUtils.getLength; import static org.smartdata.utils.FileDiffUtils.getOffset; @@ -256,7 +255,7 @@ public ScheduleResult onSchedule(CmdletInfo cmdletInfo, ActionInfo actionInfo, if (preserveAttributes != null) { action.getArgs().put(CopyFileAction.PRESERVE, preserveAttributes); } - if (rateLimiter != null) { + if (rateLimiter != null) { String strLen = getLength(fileDiff); if (strLen != null) { int appendLen = (int) (Long.parseLong(strLen) >> 20); @@ -465,74 +464,15 @@ private void runBatchInitialSync() throws MetaStoreException { } } - private List listFileStatusesOfDirs(String dirName) { - List fileStatuses = new ArrayList<>(); - try { - // We simply use local HDFS conf for getting remote file system. - // The smart file system configured for local HDFS should not be - // introduced to remote file system. - Configuration remoteConf = toRemoteClusterConfig(conf); - FileSystem fs = FileSystem.get(URI.create(dirName), remoteConf); - - FileStatus[] directoryFileStatuses = fs.listStatus(new Path(dirName)); - for (FileStatus fileStatus : directoryFileStatuses) { - // add directory - fileStatuses.add(fileStatus); - - if (!fileStatus.isDirectory()) { - continue; - } - - //all the file in this fileStatuses - // todo replace recursion with queue - List childFileStatuses = listFileStatusesOfDirs(fileStatus.getPath().getName()); - if (!childFileStatuses.isEmpty()) { - fileStatuses.addAll(childFileStatuses); - } - } - } catch (IOException e) { - LOG.debug("Fetch remote file list error!", e); - } - return fileStatuses; - } - private void initialSync(String srcDir, String destDir) throws MetaStoreException { - List srcFiles = metaStore.getFilesByPrefix(srcDir); + List srcFiles = metaStore.getFilePathsByPrefix(srcDir); LOG.info("Directory initial sync {} files", srcFiles.size()); - // - Map filesToSync = new HashMap<>(); - for (FileInfo fileInfo : srcFiles) { - // Remove prefix/parent - filesToSync.put(fileInfo.getPath().replaceFirst(srcDir, ""), fileInfo); - } - - // recursively file lists - List fileStatuses = listFileStatusesOfDirs(destDir); - if (fileStatuses.isEmpty()) { - LOG.debug("Remote directory is empty!"); - } else { - LOG.debug("Remote directory contains {} files!", fileStatuses.size()); - for (FileStatus fileStatus : fileStatuses) { - // only get file name - // todo it can be buggy because of .getPath().getName() - String destName = fileStatus.getPath().getName(); - if (filesToSync.containsKey(destName)) { - FileInfo fileInfo = filesToSync.get(destName); - String src = fileInfo.getPath(); - String dest = src.replaceFirst(srcDir, destDir); - initialSyncQueue.put(src, dest); - filesToSync.remove(destName); - } - } + for (String srcFile : srcFiles) { + String dest = srcFile.replaceFirst(srcDir, destDir); + initialSyncQueue.put(srcFile, dest); } - LOG.debug("Directory Base Sync {} files", filesToSync.size()); - for (FileInfo fileInfo : filesToSync.values()) { - String src = fileInfo.getPath(); - String dest = src.replaceFirst(srcDir, destDir); - initialSyncQueue.put(src, dest); - } runBatchInitialSync(); } @@ -562,7 +502,7 @@ private void mergePendingDiffs(String src) throws MetaStoreException { } private FileDiff runFileInitialSync(String src, String dest) throws MetaStoreException { - FileInfo srcFileInfo = metaStore.getFile(src); + BaseFileInfo srcFileInfo = metaStore.getBaseFileInfo(src); if (srcFileInfo == null || fileLocks.contains(src)) { // Primary file doesn't exist or file is syncing return null; @@ -589,7 +529,7 @@ private FileDiff runFileInitialSync(String src, String dest) throws MetaStoreExc } private FileDiff createAppendFileDiff( - FileInfo srcFileInfo, FileStatus remoteFileStatus, long copyStartOffset) { + BaseFileInfo srcFileInfo, FileStatus remoteFileStatus, long copyStartOffset) { FileDiff fileDiff = new FileDiff(FileDiffType.APPEND, FileDiffState.PENDING); fileDiff.setSrc(srcFileInfo.getPath()); diff --git a/smart-hive-support/src/main/java/org/smartdata/hive/action/HiveActionFactory.java b/smart-hive-support/src/main/java/org/smartdata/hive/action/HiveActionFactory.java index 746f8316ac..06572c3b7e 100644 --- a/smart-hive-support/src/main/java/org/smartdata/hive/action/HiveActionFactory.java +++ b/smart-hive-support/src/main/java/org/smartdata/hive/action/HiveActionFactory.java @@ -19,6 +19,7 @@ import org.smartdata.action.AbstractActionFactory; import org.smartdata.action.SmartAction; +import org.smartdata.action.SyncAction; import org.smartdata.hive.action.constraint.HmsCreateConstraintAction; import org.smartdata.hive.action.constraint.HmsDropConstraintAction; import org.smartdata.hive.action.db.HmsAlterDbAction; @@ -67,7 +68,9 @@ protected List> supportedActionClasses() { HmsAlterPartitionColumnStatsAction.class, HmsAlterTableColumnStatsAction.class, HmsDropPartitionColumnStatsAction.class, - HmsDropTableColumnStatsAction.class + HmsDropTableColumnStatsAction.class, + + SyncAction.class ); } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java index 62680d986f..3b7e6ae719 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java @@ -57,6 +57,7 @@ import org.smartdata.metrics.GeneralFileInfoSource; import org.smartdata.model.ActionInfo; import org.smartdata.model.BackUpInfo; +import org.smartdata.model.BaseFileInfo; import org.smartdata.model.CachedFileStatus; import org.smartdata.model.ClusterConfig; import org.smartdata.model.ClusterInfo; @@ -310,6 +311,17 @@ public FileInfo getFile(long fid) throws MetaStoreException { } } + public BaseFileInfo getBaseFileInfo(String path) throws MetaStoreException { + updateCache(); + try { + return generalFileInfoSource.getBaseFileInfo(path); + } catch (EmptyResultDataAccessException e) { + return null; + } catch (Exception e) { + throw new MetaStoreException(e); + } + } + public FileInfo getFile(String path) throws MetaStoreException { updateCache(); try { @@ -332,10 +344,10 @@ public List getFile() throws MetaStoreException { } } - public List getFilesByPrefix(String path) throws MetaStoreException { + public List getFilePathsByPrefix(String path) throws MetaStoreException { updateCache(); try { - return fileInfoDao.getFilesByPrefix(path); + return generalFileInfoSource.getFilePathsByPrefix(path); } catch (EmptyResultDataAccessException e) { return new ArrayList<>(); } catch (Exception e) { diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/AbstractDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/AbstractDao.java index 85fee58be9..ced1b1e631 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/AbstractDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/AbstractDao.java @@ -72,7 +72,7 @@ protected void insert( } protected int update(Map entityProperties, - String filter, Object... filterArguments) { + String filter, Object... filterArguments) { return updateInternal(entityProperties, " WHERE " + filter, filterArguments); } @@ -81,11 +81,11 @@ protected int update(Map entityProperties) { } protected int updateInternal(Map entityProperties, - String filter, Object... filterArguments) { + String filter, Object... filterArguments) { StringJoiner updateSql = new StringJoiner(", ", "UPDATE " + tableName + " SET ", filter); List setArguments = new ArrayList<>(); - for (Map.Entry property: entityProperties.entrySet()) { + for (Map.Entry property : entityProperties.entrySet()) { if (property.getValue() == null) { continue; } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java index bae24548c5..07112e768c 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/BaseFileInfoDao.java @@ -18,8 +18,10 @@ package org.smartdata.metastore.dao.impl; import lombok.Data; +import lombok.RequiredArgsConstructor; import org.smartdata.metastore.dao.AbstractDao; import org.smartdata.metrics.GeneralFileInfoSource; +import org.smartdata.model.BaseFileInfo; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @@ -55,6 +57,24 @@ public Map getPathsToIdsMapping(Collection paths) { GeneralFileInfo::getId)); } + @Override + public List getFilePathsByPrefix(String path) { + return jdbcTemplate.query( + "SELECT * FROM " + + tableName + + " WHERE path LIKE ?", + this::extractPath, path + "%"); + } + + @Override + public BaseFileInfo getBaseFileInfo(String path) { + return jdbcTemplate.queryForObject( + "SELECT * FROM " + + tableName + + " WHERE path = ?", + this::toBaseFileInfo, path); + } + private GeneralFileInfo mapRow(ResultSet resultSet, int i) throws SQLException { return new GeneralFileInfo( resultSet.getLong("fid"), @@ -62,9 +82,29 @@ private GeneralFileInfo mapRow(ResultSet resultSet, int i) throws SQLException { ); } + private String extractPath(ResultSet resultSet, int i) throws SQLException { + return resultSet.getString("path"); + } + + private BaseFileInfo toBaseFileInfo(ResultSet resultSet, int i) throws SQLException { + return new BaseFileInfoImpl( + resultSet.getString("path"), + resultSet.getLong("length"), + resultSet.getBoolean("is_dir") + ); + } + @Data private static class GeneralFileInfo { private final long id; private final String path; } + + @RequiredArgsConstructor + @Data + private static class BaseFileInfoImpl implements BaseFileInfo { + private final String path; + private final long length; + private final boolean isDir; + } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultBackUpInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultBackUpInfoDao.java index 636a7622b7..7e23a95d8e 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultBackUpInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultBackUpInfoDao.java @@ -51,20 +51,20 @@ public int getCountByRid(int rid) { @Override public BackUpInfo getByRid(long rid) { return jdbcTemplate.queryForObject("SELECT * FROM backup_file WHERE rid = ?", - new Object[] {rid}, new BackUpInfoRowMapper()); + new Object[]{rid}, new BackUpInfoRowMapper()); } @Override public List getBySrc(String src) { return jdbcTemplate.query( - "SELECT * FROM backup_file WHERE src = ?", new Object[] {src}, + "SELECT * FROM backup_file WHERE src = ?", new Object[]{src}, new BackUpInfoRowMapper()); } @Override public List getByDest(String dest) { return jdbcTemplate.query( - "SELECT * FROM backup_file WHERE dest = ?", new Object[] {dest}, + "SELECT * FROM backup_file WHERE dest = ?", new Object[]{dest}, new BackUpInfoRowMapper()); } @@ -110,14 +110,13 @@ private static class BackUpInfoRowMapper implements RowMapper { @Override public BackUpInfo mapRow(ResultSet resultSet, int i) throws SQLException { - BackUpInfo backUpInfo = new BackUpInfo(); - backUpInfo.setRid(resultSet.getLong("rid")); - backUpInfo.setSrc(resultSet.getString("src")); - backUpInfo.setDest(resultSet.getString("dest")); - backUpInfo.setPeriod(resultSet.getLong("period")); - backUpInfo.setSrcPattern(resultSet.getString("src_pattern")); - - return backUpInfo; + return BackUpInfo.builder() + .rid(resultSet.getLong("rid")) + .src(resultSet.getString("src")) + .dest(resultSet.getString("dest")) + .period(resultSet.getLong("period")) + .srcPattern(resultSet.getString("src_pattern")) + .build(); } } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java index a7c1f975a4..cdc6952136 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/impl/DefaultOzoneFileInfoDao.java @@ -27,8 +27,6 @@ import java.util.Map; public class DefaultOzoneFileInfoDao extends BaseFileInfoDao implements OzoneFileInfoDao { - private static final String TABLE_NAME = "ofile"; - private static final String FILE_ID_FIELD = "fid"; private static final String PATH_FIELD = "path"; private static final String LENGTH_FIELD = "length"; @@ -39,6 +37,7 @@ public class DefaultOzoneFileInfoDao extends BaseFileInfoDao implements OzoneFil private static final String IS_VOLUME_FIELD = "is_volume"; private static final String IS_BUCKET_FIELD = "is_bucket"; private static final String IS_S3_FIELD = "is_s3"; + private static final String IS_DIR_FIELD = "is_dir"; private static final String OWNER_FIELD = "owner"; private static final String OWNER_GROUP_FIELD = "owner_group"; private static final String PERMISSION_FIELD = "permission"; @@ -53,6 +52,11 @@ public void insert(OzoneFileInfo fileInfo) { insert(fileInfo, this::toMap); } + @Override + public void clear() { + jdbcTemplate.update("DELETE FROM ofile"); + } + @Override protected SimpleJdbcInsert simpleJdbcInsert() { return super.simpleJdbcInsert() @@ -70,6 +74,7 @@ private Map toMap(OzoneFileInfo fileInfo) { parameters.put(IS_VOLUME_FIELD, fileInfo.isVolume()); parameters.put(IS_BUCKET_FIELD, fileInfo.isBucket()); parameters.put(IS_S3_FIELD, fileInfo.isS3()); + parameters.put(IS_DIR_FIELD, fileInfo.isDir()); parameters.put(OWNER_FIELD, fileInfo.getOwner()); parameters.put(OWNER_GROUP_FIELD, fileInfo.getGroup()); parameters.put(PERMISSION_FIELD, fileInfo.getPermission()); diff --git a/smart-metastore/src/main/resources/db/changelog/changelog-12.add-ozone-file-table.xml b/smart-metastore/src/main/resources/db/changelog/changelog-12.add-ozone-file-table.xml index 51050b985c..b6b60597bd 100644 --- a/smart-metastore/src/main/resources/db/changelog/changelog-12.add-ozone-file-table.xml +++ b/smart-metastore/src/main/resources/db/changelog/changelog-12.add-ozone-file-table.xml @@ -26,6 +26,7 @@ + diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestBackUpInfoDao.java b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestBackUpInfoDao.java index effc4e7365..58c561e76b 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestBackUpInfoDao.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestBackUpInfoDao.java @@ -36,15 +36,16 @@ public void initBackUpInfoDao() { @Test public void testInsertAndGetSingleRecord() { - BackUpInfo backUpInfo = new BackUpInfo(); - backUpInfo.setRid(1); - backUpInfo.setPeriod(1); - backUpInfo.setDest(""); - backUpInfo.setSrc(""); - backUpInfo.setSrcPattern(""); - backUpInfoDao.insert(backUpInfo); + BackUpInfo backUpInfo = BackUpInfo.builder() + .rid(1) + .period(1) + .dest("") + .src("") + .srcPattern("") + .build(); - Assert.assertTrue(backUpInfoDao.getByRid(1).equals(backUpInfo)); + backUpInfoDao.insert(backUpInfo); + Assert.assertEquals(backUpInfo, backUpInfoDao.getByRid(1)); } @Test @@ -78,17 +79,22 @@ public void testBatchInsert() { @Test public void testUpdate() { - BackUpInfo backUpInfo = new BackUpInfo(); - backUpInfo.setRid(1); - backUpInfo.setSrc("test"); - backUpInfo.setDest("test"); - backUpInfo.setPeriod(1); - backUpInfo.setSrcPattern(""); + BackUpInfo backUpInfo = BackUpInfo.builder() + .rid(1) + .src("test") + .dest("test") + .period(1) + .srcPattern("") + .build(); backUpInfoDao.insert(backUpInfo); backUpInfoDao.update(1, 2); - backUpInfo.setPeriod(2); - Assert.assertTrue(backUpInfoDao.getByRid(1).equals(backUpInfo)); + + backUpInfo = backUpInfo.toBuilder() + .period(2) + .build(); + + Assert.assertEquals(backUpInfo, backUpInfoDao.getByRid(1)); } @Test diff --git a/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java b/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java index a67732f920..f8e1a5f06b 100644 --- a/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java +++ b/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java @@ -27,7 +27,6 @@ import org.smartdata.protocol.SmartClientProtocol; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.ArrayList; diff --git a/smart-ozone-support/smart-ozone/pom.xml b/smart-ozone-support/smart-ozone/pom.xml index 249b8179c1..53606bd4f1 100644 --- a/smart-ozone-support/smart-ozone/pom.xml +++ b/smart-ozone-support/smart-ozone/pom.xml @@ -39,6 +39,11 @@ smart-ozone-common 2.2.0-SNAPSHOT + + org.smartdata + smart-rule + 2.2.0-SNAPSHOT + org.smartdata smart-hadoop-action-common diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFetcherService.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFetcherService.java index 3c39974c15..cc00bfa5d7 100644 --- a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFetcherService.java +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFetcherService.java @@ -95,6 +95,8 @@ private FsObjectHandler buildFsObjectHandler() { @Override public void start() { + // todo clear the file table every time until ADH-7258 is resolved + ozoneFileInfoDao.clear(); BlockingQueue fsObjectStream = ofsSnapshotFetcher.runSnapshot(); eventStreamHandler.collectAsync(fsObjectStream); } diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java index 23c5699fdb..eb24be48b5 100644 --- a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneFileInfoDao.java @@ -21,5 +21,9 @@ import org.smartdata.ozone.model.OzoneFileInfo; public interface OzoneFileInfoDao extends GeneralFileInfoSource { + String TABLE_NAME = "ofile"; + void insert(OzoneFileInfo fileInfo); + + void clear(); } diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/action/OzoneActionFactory.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/action/OzoneActionFactory.java new file mode 100644 index 0000000000..8aac6420a9 --- /dev/null +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/action/OzoneActionFactory.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone.action; + +import org.smartdata.action.AbstractActionFactory; +import org.smartdata.action.SmartAction; +import org.smartdata.action.SyncAction; +import org.smartdata.hdfs.action.Copy2S3Action; +import org.smartdata.hdfs.action.CopyDirectoryAction; +import org.smartdata.hdfs.action.CopyFileAction; +import org.smartdata.hdfs.action.DeleteFileAction; +import org.smartdata.hdfs.action.DistCpAction; +import org.smartdata.hdfs.action.ListFileAction; +import org.smartdata.hdfs.action.MetaDataAction; +import org.smartdata.hdfs.action.ReadFileAction; +import org.smartdata.hdfs.action.RenameFileAction; +import org.smartdata.hdfs.action.WriteFileAction; + +import java.util.Arrays; +import java.util.List; + +public class OzoneActionFactory extends AbstractActionFactory { + + @Override + protected List> supportedActionClasses() { + return Arrays.asList( + CopyFileAction.class, + Copy2S3Action.class, + DeleteFileAction.class, + RenameFileAction.class, + ListFileAction.class, + ReadFileAction.class, + WriteFileAction.class, + SyncAction.class, + DistCpAction.class, + CopyDirectoryAction.class, + MetaDataAction.class + ); + } +} diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/model/OzoneFileInfo.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/model/OzoneFileInfo.java index f176cb6edd..562840745b 100644 --- a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/model/OzoneFileInfo.java +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/model/OzoneFileInfo.java @@ -20,11 +20,12 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import org.smartdata.model.BaseFileInfo; @Data @AllArgsConstructor @Builder(toBuilder = true) -public class OzoneFileInfo implements FsObjectStreamRecord { +public class OzoneFileInfo implements FsObjectStreamRecord, BaseFileInfo { private String path; private long fileId; private long length; @@ -35,6 +36,7 @@ public class OzoneFileInfo implements FsObjectStreamRecord { private boolean isVolume; private boolean isBucket; private boolean isS3; + private boolean isDir; private String owner; private String group; private short permission; diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/rule/OzoneFileObject.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/rule/OzoneFileObject.java new file mode 100644 index 0000000000..dd8ae4e7c6 --- /dev/null +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/rule/OzoneFileObject.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone.rule; + +import com.google.common.collect.ImmutableMap; +import org.smartdata.ozone.OzoneFileInfoDao; +import org.smartdata.rule.objects.ObjectType; +import org.smartdata.rule.objects.Property; +import org.smartdata.rule.objects.SmartObject; +import org.smartdata.rule.parser.ValueType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +public class OzoneFileObject extends SmartObject { + + private static final Map PROPERTIES = + ImmutableMap.builder() + .put("path", + new Property("path", ValueType.STRING, + null, OzoneFileInfoDao.TABLE_NAME, "path")) + .put("length", + new Property("length", ValueType.LONG, + null, OzoneFileInfoDao.TABLE_NAME, "length")) + .put("blocksize", + new Property("blocksize", ValueType.LONG, + null, OzoneFileInfoDao.TABLE_NAME, "block_size")) + .put("age", + new Property("age", ValueType.TIMEINTVAL, + null, OzoneFileInfoDao.TABLE_NAME, null, + "($NOW - modification_time)")) + .put("mtime", + new Property("mtime", ValueType.TIMEPOINT, + null, OzoneFileInfoDao.TABLE_NAME, "modification_time")) + .put("atime", + new Property("atime", ValueType.TIMEPOINT, + null, OzoneFileInfoDao.TABLE_NAME, "access_time")) + .put("isDir", + new Property("isDir", ValueType.BOOLEAN, + null, OzoneFileInfoDao.TABLE_NAME, "is_dir")) + .put("unsynced", + new Property("unsynced", ValueType.BOOLEAN, + null, "file_diff", null, + "state = 0")) + .put("accessCount", + new Property("accessCount", ValueType.LONG, + Collections.singletonList(ValueType.TIMEINTVAL), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("ac", + new Property("ac", ValueType.LONG, + Collections.singletonList(ValueType.TIMEINTVAL), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("accessCountTop", + new Property("accessCountTop", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("acTop", + new Property("acTop", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("accessCountBottom", + new Property("accessCountBottom", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("acBot", + new Property("acBot", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .build(); + + public OzoneFileObject() { + super(ObjectType.FILE, PROPERTIES, OzoneFileInfoDao.TABLE_NAME); + } +} \ No newline at end of file diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/rule/OzoneSmartObjectSupplier.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/rule/OzoneSmartObjectSupplier.java new file mode 100644 index 0000000000..76a34ae5ba --- /dev/null +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/rule/OzoneSmartObjectSupplier.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone.rule; + +import com.google.common.collect.ImmutableMap; +import org.smartdata.rule.objects.HmsObject; +import org.smartdata.rule.objects.SmartObject; +import org.smartdata.rule.objects.StaticMapSmartObjectSupplier; + +import java.util.Map; + +public class OzoneSmartObjectSupplier extends StaticMapSmartObjectSupplier { + private final static Map SUPPORTED_OBJECTS = ImmutableMap.of( + "file", new OzoneFileObject(), + "hms", new HmsObject() + ); + + public OzoneSmartObjectSupplier() { + super(SUPPORTED_OBJECTS); + } +} diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java index 42f12930df..f374fc951f 100644 --- a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java @@ -149,6 +149,7 @@ private CompletableFuture handleVolume(OzoneVolume volume) { private CompletableFuture handleVolume(FileStatus fileStatus) { OzoneFileInfo.Builder fileBuilder = OzoneFileInfo.builder() .path(pathWithoutAuthority(fileStatus.getPath())) + .isDir(true) .isVolume(true); return saveFile(fileStatus, fileBuilder) .thenComposeAsync(ignore -> executeInParallel( @@ -159,6 +160,7 @@ private CompletableFuture handleVolume(FileStatus fileStatus) { private CompletableFuture handleBucket(FileStatus fileStatus) { OzoneFileInfo.Builder fileBuilder = OzoneFileInfo.builder() .path(pathWithoutAuthority(fileStatus.getPath())) + .isDir(true) .isBucket(true); return saveFile(fileStatus, fileBuilder) .thenComposeAsync(ignore -> createBucketSnapshot(fileStatus), executor) @@ -172,6 +174,7 @@ private CompletableFuture handleBucket(FileStatus fileStatus) { private CompletableFuture handleKey(FileStatus fileStatus) { Path filePath = getOriginalFilePath(fileStatus.getPath()); OzoneFileInfo.Builder fileBuilder = OzoneFileInfo.builder() + .isDir(fileStatus.isDirectory()) .path(pathWithoutAuthority(filePath)); return saveFile(fileStatus, fileBuilder) .thenComposeAsync(ignore -> handleChildrenIfDirectory(fileStatus), executor); diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/DefaultSmartObjectSupplier.java b/smart-rule/src/main/java/org/smartdata/rule/objects/DefaultSmartObjectSupplier.java new file mode 100644 index 0000000000..3660a69c3a --- /dev/null +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/DefaultSmartObjectSupplier.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.rule.objects; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class DefaultSmartObjectSupplier extends StaticMapSmartObjectSupplier { + private static final Map SUPPORTED_OBJECTS = ImmutableMap.of( + "file", new FileObject(), + "hms", new HmsObject(), + "storage", new StorageObject() + ); + + public DefaultSmartObjectSupplier() { + super(SUPPORTED_OBJECTS); + } +} diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/FileObject.java b/smart-rule/src/main/java/org/smartdata/rule/objects/FileObject.java index e066943667..01976a6ca6 100644 --- a/smart-rule/src/main/java/org/smartdata/rule/objects/FileObject.java +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/FileObject.java @@ -18,11 +18,11 @@ package org.smartdata.rule.objects; +import com.google.common.collect.ImmutableMap; import org.smartdata.rule.parser.ValueType; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Map; /** @@ -30,85 +30,79 @@ */ public class FileObject extends SmartObject { - public static final Map PROPERTIES; - - static { - PROPERTIES = new HashMap<>(); - PROPERTIES.put("path", - new Property("path", ValueType.STRING, null, "file", "path")); - PROPERTIES.put("accessCount", - new Property("accessCount", ValueType.LONG, - Collections.singletonList(ValueType.TIMEINTVAL), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("ac", - new Property("ac", ValueType.LONG, - Collections.singletonList(ValueType.TIMEINTVAL), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("accessCountTop", - new Property("accessCountTop", ValueType.LONG, - Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("acTop", - new Property("acTop", ValueType.LONG, - Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("accessCountBottom", - new Property("accessCountBottom", ValueType.LONG, - Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("acBot", - new Property("acBot", ValueType.LONG, - Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("acTopSp", - new Property("acTopSp", ValueType.LONG, - Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG, ValueType.STRING), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("acBotSp", - new Property("acBotSp", ValueType.LONG, - Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG, ValueType.STRING), - "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")); - PROPERTIES.put("length", - new Property("length", ValueType.LONG, - null, "file", "length")); - PROPERTIES.put("blocksize", - new Property("blocksize", ValueType.LONG, - null, "file", "block_size")); - PROPERTIES.put("inCache", - new Property("inCache", ValueType.BOOLEAN, - null, "cached_file", null)); - PROPERTIES.put("age", - new Property("age", ValueType.TIMEINTVAL, - null, "file", null, - "($NOW - modification_time)")); - PROPERTIES.put("mtime", - new Property("mtime", ValueType.TIMEPOINT, - null, "file", "modification_time")); - PROPERTIES.put("atime", - new Property("atime", ValueType.TIMEPOINT, - null, "file", "access_time")); - PROPERTIES.put("storagePolicy", - new Property("storagePolicy", ValueType.STRING, - null, "file", null, - "(SELECT policy_name FROM storage_policy WHERE sid = file.sid)")); - PROPERTIES.put("unsynced", - new Property("unsynced", ValueType.BOOLEAN, - null, "file_diff", null, - "state = 0")); - PROPERTIES.put("isDir", - new Property("isDir", ValueType.BOOLEAN, - null, "file", "is_dir")); - PROPERTIES.put("ecPolicy", - new Property("ecPolicy", ValueType.STRING, - null, "file", null, - "(SELECT policy_name FROM ec_policy WHERE id = file.ec_policy_id)")); - } + private static final Map PROPERTIES = + ImmutableMap.builder() + .put("path", + new Property("path", ValueType.STRING, null, "file", "path")) + .put("accessCount", + new Property("accessCount", ValueType.LONG, + Collections.singletonList(ValueType.TIMEINTVAL), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("ac", + new Property("ac", ValueType.LONG, + Collections.singletonList(ValueType.TIMEINTVAL), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("accessCountTop", + new Property("accessCountTop", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("acTop", + new Property("acTop", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("accessCountBottom", + new Property("accessCountBottom", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("acBot", + new Property("acBot", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("acTopSp", + new Property("acTopSp", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG, ValueType.STRING), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("acBotSp", + new Property("acBotSp", ValueType.LONG, + Arrays.asList(ValueType.TIMEINTVAL, ValueType.LONG, ValueType.STRING), + "VIRTUAL_ACCESS_COUNT_TABLE", "", "count")) + .put("length", + new Property("length", ValueType.LONG, + null, "file", "length")) + .put("blocksize", + new Property("blocksize", ValueType.LONG, + null, "file", "block_size")) + .put("inCache", + new Property("inCache", ValueType.BOOLEAN, + null, "cached_file", null)) + .put("age", + new Property("age", ValueType.TIMEINTVAL, + null, "file", null, + "($NOW - modification_time)")) + .put("mtime", + new Property("mtime", ValueType.TIMEPOINT, + null, "file", "modification_time")) + .put("atime", + new Property("atime", ValueType.TIMEPOINT, + null, "file", "access_time")) + .put("storagePolicy", + new Property("storagePolicy", ValueType.STRING, + null, "file", null, + "(SELECT policy_name FROM storage_policy WHERE sid = file.sid)")) + .put("unsynced", + new Property("unsynced", ValueType.BOOLEAN, + null, "file_diff", null, + "state = 0")) + .put("isDir", + new Property("isDir", ValueType.BOOLEAN, + null, "file", "is_dir")) + .put("ecPolicy", + new Property("ecPolicy", ValueType.STRING, + null, "file", null, + "(SELECT policy_name FROM ec_policy WHERE id = file.ec_policy_id)")) + .build(); public FileObject() { - super(ObjectType.FILE, "file"); - } - - public Map getProperties() { - return PROPERTIES; + super(ObjectType.FILE, PROPERTIES, "file"); } } diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/HmsObject.java b/smart-rule/src/main/java/org/smartdata/rule/objects/HmsObject.java index 2c99e69f63..f7ccc1c967 100644 --- a/smart-rule/src/main/java/org/smartdata/rule/objects/HmsObject.java +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/HmsObject.java @@ -18,10 +18,10 @@ package org.smartdata.rule.objects; +import com.google.common.collect.ImmutableMap; import org.smartdata.rule.parser.ValueType; import java.util.Collections; -import java.util.HashMap; import java.util.Map; /** @@ -29,27 +29,19 @@ */ public class HmsObject extends SmartObject { - public static final Map PROPERTIES; - - static { - PROPERTIES = new HashMap<>(); - PROPERTIES.put("name", - new Property( - "name", - ValueType.STRING, - Collections.singletonList(ValueType.STRING), - "hive_metastore_event", - "entity_name", - "(entity_name LIKE SUBSTRING($0 FROM 1 FOR " - + "NULLIF(POSITION('.' IN $0), 0) - 1)) or entity_name", - true)); - } + private static final Map PROPERTIES = ImmutableMap.of( + "name", new Property( + "name", + ValueType.STRING, + Collections.singletonList(ValueType.STRING), + "hive_metastore_event", + "entity_name", + "(entity_name LIKE SUBSTRING($0 FROM 1 FOR " + + "NULLIF(POSITION('.' IN $0), 0) - 1)) or entity_name", + true) + ); public HmsObject() { - super(ObjectType.HMS, "hive_metastore_event"); - } - - public Map getProperties() { - return PROPERTIES; + super(ObjectType.HMS, PROPERTIES, "hive_metastore_event"); } } diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObject.java b/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObject.java index 9408989668..b291c57583 100644 --- a/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObject.java +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObject.java @@ -30,25 +30,10 @@ public abstract class SmartObject { private final ObjectType type; + private final Map properties; private final String baseTableName; - public static SmartObject getInstance(String typeName) { - // TODO: create through class name - switch (typeName) { - case "file": - return new FileObject(); - case "storage": - return new StorageObject(); - case "hms": - return new HmsObject(); - default: - return null; - } - } - public Property getProperty(String propertyName) { - return getProperties().get(propertyName); + return properties.get(propertyName); } - - public abstract Map getProperties(); } diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObjectSupplier.java b/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObjectSupplier.java new file mode 100644 index 0000000000..b4883076bd --- /dev/null +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/SmartObjectSupplier.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.rule.objects; + +public interface SmartObjectSupplier { + SmartObject get(String name); +} diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/StaticMapSmartObjectSupplier.java b/smart-rule/src/main/java/org/smartdata/rule/objects/StaticMapSmartObjectSupplier.java new file mode 100644 index 0000000000..95cbc966a5 --- /dev/null +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/StaticMapSmartObjectSupplier.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.rule.objects; + +import lombok.RequiredArgsConstructor; + +import java.util.Map; + +@RequiredArgsConstructor +public class StaticMapSmartObjectSupplier implements SmartObjectSupplier { + private final Map objects; + + @Override + public SmartObject get(String name) { + return objects.get(name); + } +} diff --git a/smart-rule/src/main/java/org/smartdata/rule/objects/StorageObject.java b/smart-rule/src/main/java/org/smartdata/rule/objects/StorageObject.java index f2d8f28764..a1e6197d40 100644 --- a/smart-rule/src/main/java/org/smartdata/rule/objects/StorageObject.java +++ b/smart-rule/src/main/java/org/smartdata/rule/objects/StorageObject.java @@ -17,36 +17,32 @@ */ package org.smartdata.rule.objects; +import com.google.common.collect.ImmutableMap; import org.smartdata.rule.parser.ValueType; import java.util.Collections; -import java.util.HashMap; import java.util.Map; /** * Definition of rule object 'Storage'. */ public class StorageObject extends SmartObject { - public static final Map PROPERTIES; - - static { - PROPERTIES = new HashMap<>(); - PROPERTIES.put("capacity", new Property("capacity", ValueType.LONG, - Collections.singletonList(ValueType.STRING), "storage", "capacity", - "type = $0 AND capacity")); - PROPERTIES.put("free", new Property("free", ValueType.LONG, - Collections.singletonList(ValueType.STRING), "storage", "free", - "type = $0 AND free")); - PROPERTIES.put("utilization", new Property("utilization", ValueType.LONG, - Collections.singletonList(ValueType.STRING), "storage", "free", - "type = $0 AND (capacity - free) * 100.0 / capacity")); - } + public static final Map PROPERTIES = ImmutableMap.of( + "capacity", + new Property("capacity", ValueType.LONG, + Collections.singletonList(ValueType.STRING), "storage", "capacity", + "type = $0 AND capacity"), + "free", + new Property("free", ValueType.LONG, + Collections.singletonList(ValueType.STRING), "storage", "free", + "type = $0 AND free"), + "utilization", + new Property("utilization", ValueType.LONG, + Collections.singletonList(ValueType.STRING), "storage", "free", + "type = $0 AND (capacity - free) * 100.0 / capacity") + ); public StorageObject() { - super(ObjectType.STORAGE, "storage"); - } - - public Map getProperties() { - return PROPERTIES; + super(ObjectType.STORAGE, PROPERTIES, "storage"); } } diff --git a/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleStringParser.java b/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleStringParser.java index fc29460bff..01b0ca296c 100644 --- a/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleStringParser.java +++ b/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleStringParser.java @@ -17,6 +17,7 @@ */ package org.smartdata.rule.parser; +import com.google.common.collect.ImmutableMap; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CommonTokenStream; @@ -29,61 +30,48 @@ import org.smartdata.exception.SsmParseException; import org.smartdata.model.CmdletDescriptor; import org.smartdata.model.rule.RuleTranslationResult; +import org.smartdata.rule.objects.SmartObjectSupplier; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; /** Parser a rule string and translate it. */ public class SmartRuleStringParser { - private String rule; - private TranslationContext ctx = null; - private SmartConf conf; - - private static Map optCond = new HashMap<>(); - - static { - optCond.put("allssd", "storagePolicy != \"ALL_SSD\""); - optCond.put("onessd", "storagePolicy != \"ONE_SSD\""); - optCond.put("archive", "storagePolicy != \"COLD\""); - optCond.put("alldisk", "storagePolicy != \"HOT\""); - optCond.put("onedisk", "storagePolicy != \"WARM\""); - optCond.put("ramdisk", "storagePolicy != \"LAZY_PERSIST\""); - optCond.put("cache", "not inCache"); - optCond.put("uncache", "inCache"); - optCond.put("sync", "unsynced"); - optCond.put("ec", "1"); - optCond.put("unec", "1"); - } + private static final Map CONDITION_REWRITES = + ImmutableMap.builder() + .put("allssd", "storagePolicy != \"ALL_SSD\"") + .put("onessd", "storagePolicy != \"ONE_SSD\"") + .put("archive", "storagePolicy != \"COLD\"") + .put("alldisk", "storagePolicy != \"HOT\"") + .put("onedisk", "storagePolicy != \"WARM\"") + .put("ramdisk", "storagePolicy != \"LAZY_PERSIST\"") + .put("cache", "not inCache") + .put("uncache", "inCache") + .put("sync", "unsynced") + .put("ec", "1") + .put("unec", "1") + .build(); - List parseErrors = new ArrayList(); - String parserErrorMessage = ""; - public class SSMRuleErrorListener extends BaseErrorListener { - @Override - public void syntaxError( - Recognizer recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - List stack = ((Parser) recognizer).getRuleInvocationStack(); - Collections.reverse(stack); - parserErrorMessage += "Line " + line + ", Char " + charPositionInLine + " : " + msg + "\n"; - parseErrors.add(e); - } - } + private final String rule; + private final SmartConf conf; + private final TranslationContext ctx; + private final SmartObjectSupplier smartObjectSupplier; + private final List parseErrors = new ArrayList<>(); + + private String parserErrorMessage = ""; - public SmartRuleStringParser(String rule, TranslationContext ctx, SmartConf conf) { + public SmartRuleStringParser(String rule, TranslationContext ctx, + SmartObjectSupplier smartObjectSupplier, SmartConf conf) { this.rule = rule; this.ctx = ctx; this.conf = conf; + this.smartObjectSupplier = smartObjectSupplier; } public RuleTranslationResult translate() throws IOException { @@ -93,11 +81,11 @@ public RuleTranslationResult translate() throws IOException { throw new IOException("No cmdlet specified in Rule"); } String actName = cmdDes.getActionName(0); - if (cmdDes.getActionSize() != 1 || optCond.get(actName) == null) { + if (cmdDes.getActionSize() != 1 || CONDITION_REWRITES.get(actName) == null) { return tr; } - String repl = optCond.get(actName); + String repl = CONDITION_REWRITES.get(actName); if (cmdDes.getActionName(0).equals("ec") || cmdDes.getActionName(0).equals("unec")) { String policy; if (cmdDes.getActionName(0).equals("ec")) { @@ -134,13 +122,30 @@ private RuleTranslationResult doTranslate(String rule) throws IOException { throw new SsmParseException(parserErrorMessage); } - SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator(ctx); + SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator(ctx, smartObjectSupplier); try { visitor.visit(tree); } catch (RuntimeException e) { - throw new SsmParseException(e.getMessage()); + throw new SsmParseException(e.getMessage(), e); } return visitor.generateSql(); } + + public class SSMRuleErrorListener extends BaseErrorListener { + @Override + public void syntaxError( + Recognizer recognizer, + Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + List stack = ((Parser) recognizer).getRuleInvocationStack(); + Collections.reverse(stack); + parserErrorMessage += "Line " + line + ", Char " + charPositionInLine + " : " + msg + "\n"; + parseErrors.add(e); + } + } } + diff --git a/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleVisitTranslator.java b/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleVisitTranslator.java index b1ae26679e..393c0da3a3 100644 --- a/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleVisitTranslator.java +++ b/smart-rule/src/main/java/org/smartdata/rule/parser/SmartRuleVisitTranslator.java @@ -31,6 +31,7 @@ import org.smartdata.rule.objects.Property; import org.smartdata.rule.objects.PropertyRealParas; import org.smartdata.rule.objects.SmartObject; +import org.smartdata.rule.objects.SmartObjectSupplier; import org.smartdata.rule.parser.SmartRuleParser.TimeintvalexprContext; import org.smartdata.utils.StringUtil; @@ -58,6 +59,7 @@ public class SmartRuleVisitTranslator extends SmartRuleBaseVisitor { private final TranslationContext transCtx; private final CmdletParser cmdletParser; + private final SmartObjectSupplier smartObjectSupplier; private TreeNode objFilter; private TreeNode conditions; @@ -67,12 +69,14 @@ public class SmartRuleVisitTranslator extends SmartRuleBaseVisitor { private int[] condPosition; private long minTimeInterval; - public SmartRuleVisitTranslator() { - this(null); + public SmartRuleVisitTranslator(SmartObjectSupplier smartObjectSupplier) { + this(null, smartObjectSupplier); } - public SmartRuleVisitTranslator(TranslationContext transCtx) { + public SmartRuleVisitTranslator(TranslationContext transCtx, + SmartObjectSupplier smartObjectSupplier) { this.transCtx = transCtx; + this.smartObjectSupplier = smartObjectSupplier; this.objects = new HashMap<>(); this.pathCheckGlob = new ArrayList<>(); this.minTimeInterval = Long.MAX_VALUE; @@ -82,18 +86,18 @@ public SmartRuleVisitTranslator(TranslationContext transCtx) { @Override public TreeNode visitObjTypeOnly(SmartRuleParser.ObjTypeOnlyContext ctx) { String objName = ctx.OBJECTTYPE().getText(); - SmartObject obj = SmartObject.getInstance(objName); - objects.put(objName, obj); - objects.put("Default", obj); + SmartObject object = smartObjectSupplier.get(objName); + objects.put(objName, object); + objects.put("Default", object); return null; } @Override public TreeNode visitObjTypeWith(SmartRuleParser.ObjTypeWithContext ctx) { String objName = ctx.OBJECTTYPE().getText(); - SmartObject obj = SmartObject.getInstance(objName); - objects.put(objName, obj); - objects.put("Default", obj); + SmartObject object = smartObjectSupplier.get(objName); + objects.put(objName, object); + objects.put("Default", object); objFilter = visit(ctx.objfilter()); return null; } @@ -270,12 +274,7 @@ public TreeNode visitTieTiIdExpr(SmartRuleParser.TieTiIdExprContext ctx) { } private SmartObject createIfNotExist(String objName) { - SmartObject obj = objects.get(objName); - if (obj == null) { - obj = SmartObject.getInstance(objName); - objects.put(objName, obj); - } - return obj; + return objects.computeIfAbsent(objName, smartObjectSupplier::get); } // ID @@ -682,10 +681,10 @@ public RuleTranslationResult generateSql() throws IOException { switch (object.getType()) { case DIRECTORY: case FILE: - ret = "SELECT path FROM file"; + ret = "SELECT path FROM " + object.getBaseTableName(); break; case HMS: - ret = "SELECT id FROM hive_metastore_event"; + ret = "SELECT id FROM " + object.getBaseTableName(); break; default: throw new IOException( diff --git a/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleParser.java b/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleParser.java index 548abab932..cdd28ca5f3 100644 --- a/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleParser.java +++ b/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleParser.java @@ -27,6 +27,7 @@ import org.junit.Assert; import org.junit.Test; import org.smartdata.model.rule.RuleTranslationResult; +import org.smartdata.rule.objects.DefaultSmartObjectSupplier; import org.smartdata.rule.parser.SmartRuleLexer; import org.smartdata.rule.parser.SmartRuleParser; import org.smartdata.rule.parser.SmartRuleVisitTranslator; @@ -123,7 +124,8 @@ private void parseAndExecuteRule(String rule) throws Exception { System.out.println("Parser tree: " + tree.toStringTree(parser)); System.out.println("Total number of errors: " + parseErrors.size()); - SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator(); + SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator( + new DefaultSmartObjectSupplier()); visitor.visit(tree); System.out.println("\nQuery:"); diff --git a/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleStringParser.java b/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleStringParser.java index 7b942a0905..663556841a 100644 --- a/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleStringParser.java +++ b/smart-rule/src/test/java/org/smartdata/rule/TestSmartRuleStringParser.java @@ -20,6 +20,7 @@ import org.junit.Test; import org.smartdata.conf.SmartConf; import org.smartdata.model.rule.RuleTranslationResult; +import org.smartdata.rule.objects.DefaultSmartObjectSupplier; import org.smartdata.rule.parser.SmartRuleStringParser; import org.smartdata.rule.parser.TranslationContext; @@ -50,7 +51,8 @@ public void testRuleTranslate() throws Exception { private void parseRule(String rule) throws Exception { TranslationContext tc = new TranslationContext(1, System.currentTimeMillis()); - SmartRuleStringParser parser = new SmartRuleStringParser(rule, tc, new SmartConf()); + SmartRuleStringParser parser = new SmartRuleStringParser(rule, tc, + new DefaultSmartObjectSupplier(), new SmartConf()); RuleTranslationResult tr = parser.translate(); int index = 1; diff --git a/smart-server/src/test/java/org/smartdata/server/engine/audit/TestRuleLifecycleLogger.java b/smart-server/src/test/java/org/smartdata/server/engine/audit/TestRuleLifecycleLogger.java index 7138b0bfdc..409ae2aa1e 100644 --- a/smart-server/src/test/java/org/smartdata/server/engine/audit/TestRuleLifecycleLogger.java +++ b/smart-server/src/test/java/org/smartdata/server/engine/audit/TestRuleLifecycleLogger.java @@ -67,6 +67,7 @@ public void init() throws Exception { serverContext, null, auditService, new ActionRegistry(fsContext.actionFactories()), principalManager, + fsContext.smartObjectSupplier(), fsContext.ruleExecutorPlugins(serverContext, null)); ruleManager.init(); ruleManager.start(); diff --git a/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java b/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java index e2e1e19456..e40c57f470 100644 --- a/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java +++ b/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java @@ -65,6 +65,7 @@ public void init() throws Exception { new NoOpAuditService(), new ActionRegistry(fsContext.actionFactories()), principalManager, + fsContext.smartObjectSupplier(), fsContext.ruleExecutorPlugins(serverContext, null)); ruleManager.init(); ruleManager.start(); diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java index 9e022b366c..34b04e0383 100644 --- a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java @@ -65,6 +65,7 @@ public void initialize(ConfigurableApplicationContext applicationContext) { "smartPrincipalManager", smartEngine.getSmartPrincipalManager()); beanFactory.registerSingleton("dbFileAccessManager", smartEngine.getFileAccessManager().getFileAccessCountManager()); + beanFactory.registerSingleton("actionRegistry", smartEngine.getActionRegistry()); } } diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/MetadataControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/MetadataControllerDelegate.java new file mode 100644 index 0000000000..2857a67ca0 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/MetadataControllerDelegate.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.controller; + +import lombok.RequiredArgsConstructor; +import org.smartdata.action.ActionMetadata; +import org.smartdata.action.ActionRegistry; +import org.smartdata.server.generated.api.MetadataApiDelegate; +import org.smartdata.server.generated.model.ActionsMetadataDto; +import org.smartdata.server.mappers.ActionMetadataMapper; +import org.springframework.stereotype.Component; + +import java.util.Set; + +@Component +@RequiredArgsConstructor +public class MetadataControllerDelegate implements MetadataApiDelegate { + + private final ActionRegistry actionRegistry; + private final ActionMetadataMapper actionMetadataMapper; + + @Override + public ActionsMetadataDto getActionsMetadata() { + Set actionMetadata = actionRegistry.getActionMetadata(); + return actionMetadataMapper.toActionsMetadataDto(actionMetadata); + } +} diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApi.java new file mode 100644 index 0000000000..53afe73215 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApi.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.generated.api; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.smartdata.server.generated.model.ActionsMetadataDto; +import org.springframework.http.HttpStatus; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseStatus; + +import javax.annotation.Generated; + +@Generated(value = "org.openapitools.codegen.languages.SpringCodegen") +@Validated +@Tag(name = "Metadata", description = "the Metadata API") +public interface MetadataApi { + + default MetadataApiDelegate getDelegate() { + return new MetadataApiDelegate() { + }; + } + + /** + * GET /api/v2/metadata/actions : List all actions metadata + * + * @return OK (status code 200) + * or Unauthorized (status code 401) + */ + @Operation( + operationId = "getActionsMetadata", + summary = "List all actions metadata", + tags = {"Metadata"}, + responses = { + @ApiResponse(responseCode = "200", description = "OK", content = { + @Content(mediaType = "application/json", schema = @Schema(implementation = ActionsMetadataDto.class)) + }), + @ApiResponse(responseCode = "401", description = "Unauthorized") + }, + security = { + @SecurityRequirement(name = "basicAuth") + } + ) + @RequestMapping( + method = RequestMethod.GET, + value = "/api/v2/metadata/actions", + produces = {"application/json"} + ) + @ResponseStatus(HttpStatus.OK) + + default ActionsMetadataDto getActionsMetadata( + + ) throws Exception { + return getDelegate().getActionsMetadata(); + } + +} diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApiController.java new file mode 100644 index 0000000000..f292eb4c1a --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApiController.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.generated.api; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Generated; + +import java.util.Optional; + +@Generated(value = "org.openapitools.codegen.languages.SpringCodegen") +@RestController +@RequestMapping("${openapi.sSMAPIDocumentation.base-path:}") +public class MetadataApiController implements MetadataApi { + + private final MetadataApiDelegate delegate; + + public MetadataApiController(@Autowired(required = false) MetadataApiDelegate delegate) { + this.delegate = Optional.ofNullable(delegate).orElse(new MetadataApiDelegate() {}); + } + + @Override + public MetadataApiDelegate getDelegate() { + return delegate; + } + +} diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApiDelegate.java new file mode 100644 index 0000000000..fad15d8de2 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/MetadataApiDelegate.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.generated.api; + +import org.smartdata.server.generated.model.ActionsMetadataDto; +import org.springframework.web.context.request.NativeWebRequest; + +import javax.annotation.Generated; + +import java.util.Optional; + +/** + * A delegate to be called by the {@link MetadataApiController}}. + * Implement this interface with a {@link org.springframework.stereotype.Service} annotated class. + */ +@Generated(value = "org.openapitools.codegen.languages.SpringCodegen") +public interface MetadataApiDelegate { + + default Optional getRequest() { + return Optional.empty(); + } + + /** + * GET /api/v2/metadata/actions : List all actions metadata + * + * @return OK (status code 200) + * or Unauthorized (status code 401) + * @see MetadataApi#getActionsMetadata + */ + default ActionsMetadataDto getActionsMetadata() throws Exception { + throw new IllegalArgumentException("Not implemented"); + + } + +} diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionMetadataDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionMetadataDto.java new file mode 100644 index 0000000000..e09380976e --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionMetadataDto.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.generated.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.annotation.Generated; +import javax.validation.constraints.NotNull; + +import java.util.Objects; + +/** + * ActionMetadataDto + */ + +@JsonTypeName("ActionMetadata") +@Generated(value = "org.openapitools.codegen.languages.SpringCodegen") +public class ActionMetadataDto { + + private String name; + + private String usage = null; + + public ActionMetadataDto() { + super(); + } + + /** + * Constructor with only required parameters + */ + public ActionMetadataDto(String name) { + this.name = name; + } + + public ActionMetadataDto name(String name) { + this.name = name; + return this; + } + + /** + * SSM host on which this action is running + * @return name + */ + @NotNull + @Schema(name = "name", description = "SSM host on which this action is running", requiredMode = Schema.RequiredMode.REQUIRED) + @JsonProperty("name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public ActionMetadataDto usage(String usage) { + this.usage = usage; + return this; + } + + /** + * SSM host on which this action is running + * @return usage + */ + + @Schema(name = "usage", description = "SSM host on which this action is running", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("usage") + public String getUsage() { + return usage; + } + + public void setUsage(String usage) { + this.usage = usage; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ActionMetadataDto actionMetadata = (ActionMetadataDto) o; + return Objects.equals(this.name, actionMetadata.name) && + Objects.equals(this.usage, actionMetadata.usage); + } + + @Override + public int hashCode() { + return Objects.hash(name, usage); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class ActionMetadataDto {\n"); + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" usage: ").append(toIndentedString(usage)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionsMetadataDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionsMetadataDto.java new file mode 100644 index 0000000000..f702c29b21 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionsMetadataDto.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.generated.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.annotation.Generated; +import javax.validation.Valid; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * ActionsMetadataDto + */ + +@JsonTypeName("ActionsMetadata") +@Generated(value = "org.openapitools.codegen.languages.SpringCodegen") +public class ActionsMetadataDto { + + @Valid + private List<@Valid ActionMetadataDto> items; + + public ActionsMetadataDto items(List<@Valid ActionMetadataDto> items) { + this.items = items; + return this; + } + + public ActionsMetadataDto addItemsItem(ActionMetadataDto itemsItem) { + if (this.items == null) { + this.items = new ArrayList<>(); + } + this.items.add(itemsItem); + return this; + } + + /** + * List of actions metadata + * @return items + */ + @Valid + @Schema(name = "items", description = "List of actions metadata", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("items") + public List<@Valid ActionMetadataDto> getItems() { + return items; + } + + public void setItems(List<@Valid ActionMetadataDto> items) { + this.items = items; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ActionsMetadataDto actionsMetadata = (ActionsMetadataDto) o; + return Objects.equals(this.items, actionsMetadata.items); + } + + @Override + public int hashCode() { + return Objects.hash(items); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class ActionsMetadataDto {\n"); + sb.append(" items: ").append(toIndentedString(items)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ActionMetadataMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ActionMetadataMapper.java new file mode 100644 index 0000000000..046bace275 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ActionMetadataMapper.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server.mappers; + +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; +import org.smartdata.action.ActionMetadata; +import org.smartdata.server.generated.model.ActionMetadataDto; +import org.smartdata.server.generated.model.ActionsMetadataDto; + +import java.util.List; +import java.util.Set; + + +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.ERROR) +public interface ActionMetadataMapper extends SmartMapper { + + List toActionMetadataDtos(Set metadataSet); + + default ActionsMetadataDto toActionsMetadataDto(Set metadataSet) { + return new ActionsMetadataDto() + .items(toActionMetadataDtos(metadataSet)); + } +} diff --git a/smart-web-server/smart-master-web-server/src/main/resources/api/resources/actions-metadata.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/actions-metadata.yaml new file mode 100644 index 0000000000..298a0bf137 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/actions-metadata.yaml @@ -0,0 +1,21 @@ +get: + tags: + - Metadata + summary: List all actions metadata + operationId: getActionsMetadata + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + title: ActionsMetadata + properties: + items: + type: array + description: List of actions metadata + items: + $ref: '../schemas/metadata/ActionMetadata.yaml' + '401': + description: Unauthorized diff --git a/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/metadata/ActionMetadata.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/metadata/ActionMetadata.yaml new file mode 100644 index 0000000000..126a2ea191 --- /dev/null +++ b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/metadata/ActionMetadata.yaml @@ -0,0 +1,13 @@ +title: ActionMetadata +type: object +properties: + name: + type: string + description: SSM host on which this action is running + usage: + type: string + nullable: true + default: null + description: SSM host on which this action is running +required: + - name diff --git a/smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml index ad730787a2..ae23d641bf 100644 --- a/smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml +++ b/smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml @@ -11,6 +11,7 @@ tags: - name: Cluster - name: Audit - name: System + - name: Metadata servers: - url: http://localhost:8081 @@ -61,6 +62,10 @@ paths: /api/v2/system/current-user: $ref: './resources/current-user.yaml' + # Metadata + /api/v2/metadata/actions: + $ref: './resources/actions-metadata.yaml' + components: securitySchemes: basicAuth: diff --git a/smart-web-server/smart-master-web-server/src/main/resources/static/ssm-api.yaml b/smart-web-server/smart-master-web-server/src/main/resources/static/ssm-api.yaml index 7ad5369cb9..e98cf1242a 100644 --- a/smart-web-server/smart-master-web-server/src/main/resources/static/ssm-api.yaml +++ b/smart-web-server/smart-master-web-server/src/main/resources/static/ssm-api.yaml @@ -16,6 +16,7 @@ tags: - name: Cluster - name: Audit - name: System +- name: Metadata paths: /api/v2/rules: get: @@ -770,6 +771,21 @@ paths: summary: Get current logged in user tags: - System + /api/v2/metadata/actions: + get: + operationId: getActionsMetadata + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ActionsMetadata' + description: OK + "401": + description: Unauthorized + summary: List all actions metadata + tags: + - Metadata components: parameters: page-request: @@ -1638,6 +1654,22 @@ components: - name title: UserInfo type: object + ActionMetadata: + example: + usage: usage + name: name + properties: + name: + description: SSM host on which this action is running + type: string + usage: + description: SSM host on which this action is running + nullable: true + type: string + required: + - name + title: ActionMetadata + type: object ExecutorType: description: Type of the cmdlet executor enum: @@ -1984,6 +2016,21 @@ components: timestamp: 1 objectType: null title: AuditEvents + ActionsMetadata: + example: + items: + - usage: usage + name: name + - usage: usage + name: name + properties: + items: + description: List of actions metadata + items: + $ref: '#/components/schemas/ActionMetadata' + type: array + title: ActionsMetadata + type: object securitySchemes: basicAuth: scheme: basic diff --git a/supports/tools/docker/ozone/conf/smart-site.xml b/supports/tools/docker/ozone/conf/smart-site.xml index 5443d8f348..77bb17ab52 100755 --- a/supports/tools/docker/ozone/conf/smart-site.xml +++ b/supports/tools/docker/ozone/conf/smart-site.xml @@ -47,4 +47,12 @@ smart.rest.server.security.enabled false + + ozone.replication + ONE + + + ozone.replication.type + RATIS + \ No newline at end of file diff --git a/supports/tools/docker/ozone/docker-compose.yaml b/supports/tools/docker/ozone/docker-compose.yaml index 6e60a8dee0..c28bf7aba3 100644 --- a/supports/tools/docker/ozone/docker-compose.yaml +++ b/supports/tools/docker/ozone/docker-compose.yaml @@ -27,11 +27,12 @@ x-common-config: OZONE-SITE.XML_ozone.om.http-address: "om:9874" OZONE-SITE.XML_ozone.recon.address: "recon:9891" OZONE-SITE.XML_ozone.recon.db.dir: "/data/metadata/recon" - OZONE-SITE.XML_ozone.replication: "1" OZONE-SITE.XML_ozone.scm.block.client.address: "scm" OZONE-SITE.XML_ozone.scm.client.address: "scm" OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" OZONE-SITE.XML_ozone.scm.names: "scm" + OZONE-SITE.XML_ozone.replication: "ONE" + OZONE-SITE.XML_ozone.replication.type: "RATIS" no_proxy: "om,recon,scm,s3g,localhost,127.0.0.1" version: "3"