From 36cbd1445f542876418825fd939a1732d3cd6ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E8=B4=B5?= Date: Fri, 15 May 2015 17:15:56 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E8=8A=82=E7=82=B9=20=E5=90=AF=E7=94=A8/?= =?UTF-8?q?=E7=A6=81=E7=94=A8=20=E7=9B=B8=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- job-admin/pom.xml | 62 ++++++++++ .../web/controller/AbstractController.java | 9 ++ .../job/web/controller/NodeController.java | 12 ++ .../lts/job/web/support/AppConfigurer.java | 69 +++++++++++ .../lts/job/web/support/node/NodeManager.java | 38 ++++++ .../job/web/support/node/ZkNodeManager.java | 62 ++++++++++ .../src/main/resources/config.properties | 3 + .../src/main/resources/spring-servlet.xml | 42 +++++++ .../java/com/lts/job/client/JobClient.java | 23 ++++ .../java/com/lts/job/core/Application.java | 25 ++-- .../job/core/cluster/AbstractClientNode.java | 5 +- .../lts/job/core/cluster/AbstractJobNode.java | 25 ++++ .../job/core/cluster/AbstractServerNode.java | 4 +- .../lts/job/core/cluster/MasterElector.java | 2 +- ...anager.java => SubscribedNodeManager.java} | 6 +- .../com/lts/job/core/constant/EcTopic.java | 14 +++ .../java/com/lts/job/core/domain/Job.java | 13 +- .../lts/job/core/domain/JobNodeConfig.java | 12 +- .../core/exception/JobSubmitException.java | 27 ++++ .../factory}/NamedThreadFactory.java | 3 +- .../job/core/listener/SelfChangeListener.java | 56 +++++++++ .../lts/job/core/registry/ZkNodeRegistry.java | 8 +- .../lts/job/core/registry/ZkNodeUtils.java | 114 +++++++++++++++++ .../lts/job/core/registry/ZkPathParser.java | 115 ++---------------- .../job/core/remoting/HeartBeatMonitor.java | 2 +- .../main/java/com/lts/job/ec/EventCenter.java | 47 +++++++ .../main/java/com/lts/job/ec/EventInfo.java | 51 ++++++++ .../java/com/lts/job/ec/EventSubscriber.java | 33 +++++ .../java/com/lts/job/ec/JvmEventCenter.java | 85 +++++++++++++ .../main/java/com/lts/job/ec/Observer.java | 11 ++ .../com/lts/job/task/tracker/TaskTracker.java | 26 ++++ .../job/task/tracker/runner/RunnerPool.java | 14 +++ .../task/tracker/ThreadPoolDynamicTest.java | 63 ++++++++++ .../java/com/lts/job/tracker/JobTracker.java | 11 ++ .../tracker/processor/JobPullProcessor.java | 5 +- .../support/checker/DeadJobChecker.java | 2 +- pom.xml | 2 +- 37 files changed, 969 insertions(+), 132 deletions(-) create mode 100644 job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java create mode 100644 job-admin/src/main/java/com/lts/job/web/controller/NodeController.java create mode 100644 job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java create mode 100644 job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java create mode 100644 job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java create mode 100644 job-admin/src/main/resources/config.properties create mode 100644 job-admin/src/main/resources/spring-servlet.xml rename job-core/src/main/java/com/lts/job/core/cluster/{NodeManager.java => SubscribedNodeManager.java} (95%) create mode 100644 job-core/src/main/java/com/lts/job/core/constant/EcTopic.java create mode 100644 job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java rename job-core/src/main/java/com/lts/job/{registry/util => core/factory}/NamedThreadFactory.java (94%) create mode 100644 job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java create mode 100644 job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java create mode 100644 job-core/src/main/java/com/lts/job/ec/EventCenter.java create mode 100644 job-core/src/main/java/com/lts/job/ec/EventInfo.java create mode 100644 job-core/src/main/java/com/lts/job/ec/EventSubscriber.java create mode 100644 job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java create mode 100644 job-core/src/main/java/com/lts/job/ec/Observer.java create mode 100644 job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java diff --git a/job-admin/pom.xml b/job-admin/pom.xml index b11677461..47cc9eafd 100644 --- a/job-admin/pom.xml +++ b/job-admin/pom.xml @@ -11,5 +11,67 @@ war job-admin + + 3.2.4.RELEASE + + + + + org.apache.tomcat.maven + tomcat7-maven-plugin + 2.0 + + / + 8080 + tomcat + lts-admin.war + utf-8 + + + + + + + + com.lts + job-core + ${project.version} + + + org.springframework + spring-beans + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-context-support + ${spring.version} + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-web + ${spring.version} + + + org.springframework + spring-webmvc + ${spring.version} + + \ No newline at end of file diff --git a/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java b/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java new file mode 100644 index 000000000..0778ac593 --- /dev/null +++ b/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java @@ -0,0 +1,9 @@ +package com.lts.job.web.controller; + +/** + * Created by hugui on 5/9/15. + */ +public class AbstractController { + + +} diff --git a/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java b/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java new file mode 100644 index 000000000..3169d3182 --- /dev/null +++ b/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java @@ -0,0 +1,12 @@ +package com.lts.job.web.controller; + +import org.springframework.stereotype.Controller; + +/** + * Created by hugui on 5/11/15. + */ +@Controller +public class NodeController extends AbstractController{ + + +} diff --git a/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java b/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java new file mode 100644 index 000000000..0084d3101 --- /dev/null +++ b/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java @@ -0,0 +1,69 @@ +package com.lts.job.web.support; + +import java.util.*; + +/** + * Created by hugui on 5/11/15. + */ +public class AppConfigurer { + + private static final Map CONFIG = new HashMap(); + + static { + try { + List configList = new ArrayList(2); + configList.add("config"); + Locale locale = Locale.getDefault(); + for (String config : configList) { + try { + ResourceBundle localResource = ResourceBundle.getBundle(config, locale); + for (Object o : localResource.keySet()) { + String key = o.toString(); + String value = localResource.getString(key); + CONFIG.put(key, value); + } + } catch (MissingResourceException e) { + // ignore + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static String getProperties(String name) { + return CONFIG.get(name); + } + + public static String getProperties(String name, String defaultValue) { + String returnValue = CONFIG.get(name); + if (returnValue == null || returnValue.equals("")) { + returnValue = defaultValue; + } + return returnValue; + } + + public static int getInteger(String name, int defaultValue) { + String returnValue = CONFIG.get(name); + if (returnValue == null || returnValue.equals("")) { + return defaultValue; + } + return Integer.parseInt(returnValue.trim()); + } + + public static int getInteger(String name) { + return Integer.parseInt(CONFIG.get(name)); + } + + public static boolean getBoolean(String name, boolean defaultValue) { + String returnValue = CONFIG.get(name); + if (returnValue == null || returnValue.equals("")) { + return defaultValue; + } + return Boolean.valueOf(CONFIG.get(name)); + } + + public static boolean getBoolean(String name) { + return Boolean.valueOf(CONFIG.get(name)); + } +} diff --git a/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java b/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java new file mode 100644 index 000000000..4f8616a50 --- /dev/null +++ b/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java @@ -0,0 +1,38 @@ +package com.lts.job.web.support.node; + +import com.lts.job.core.cluster.Node; +import com.lts.job.core.cluster.NodeType; + +import java.util.List; +import java.util.Map; + +/** + * Created by hugui on 5/11/15. + */ +public interface NodeManager { + + /** + * 得到所有节点 + * @param cluster + * @return + */ + public Map> getAllNodes(String cluster); + + /** + * 启用节点 + * @param node + */ + public void enableNode(Node node); + + /** + * 禁用节点 + * @param node + */ + public void disableNode(Node node); + + /** + * 更改节点的工作线程 + * @param node + */ + public void updateWorkThreads(Node node); +} diff --git a/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java b/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java new file mode 100644 index 000000000..fdb94bb64 --- /dev/null +++ b/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java @@ -0,0 +1,62 @@ +package com.lts.job.web.support.node; + +import com.lts.job.core.cluster.Node; +import com.lts.job.core.cluster.NodeType; +import com.lts.job.core.registry.ZkNodeUtils; +import com.lts.job.core.util.CollectionUtils; +import com.lts.job.registry.zookeeper.ZookeeperClient; +import com.lts.job.registry.zookeeper.zkclient.ZkClientZookeeperClient; +import com.lts.job.web.support.AppConfigurer; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by hugui on 5/11/15. + */ +public class ZkNodeManager implements NodeManager { + + private ZookeeperClient zkClient; + + public ZkNodeManager() { + this.zkClient = new ZkClientZookeeperClient(AppConfigurer.getProperties("zookeeper.address")); + } + + @Override + public Map> getAllNodes(String clusterName) { + String basePath = ZkNodeUtils.getBasePath(clusterName); + List nodeTypes = zkClient.getChildren(basePath); + Map> nodeMap = new HashMap>(); + if (CollectionUtils.isNotEmpty(nodeTypes)) { + for (String nodeType : nodeTypes) { + List nodes = zkClient.getChildren(basePath + "/" + nodeType); + if (CollectionUtils.isEmpty(nodes)) { + List nodeList = new ArrayList(nodes.size()); + for (String node : nodes) { + nodeList.add(ZkNodeUtils.parse(clusterName, node)); + } + nodeMap.put(NodeType.valueOf(nodeType), nodeList); + } + } + } + return nodeMap; + } + + @Override + public void enableNode(Node node) { + + } + + @Override + public void disableNode(Node node) { + + } + + @Override + public void updateWorkThreads(Node node) { + + } + +} diff --git a/job-admin/src/main/resources/config.properties b/job-admin/src/main/resources/config.properties new file mode 100644 index 000000000..f4c8aa828 --- /dev/null +++ b/job-admin/src/main/resources/config.properties @@ -0,0 +1,3 @@ + +#zookeeper地址 +zookeeper.address=127.0.0.1:2181 \ No newline at end of file diff --git a/job-admin/src/main/resources/spring-servlet.xml b/job-admin/src/main/resources/spring-servlet.xml new file mode 100644 index 000000000..beb61098d --- /dev/null +++ b/job-admin/src/main/resources/spring-servlet.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/job-client/src/main/java/com/lts/job/client/JobClient.java b/job-client/src/main/java/com/lts/job/client/JobClient.java index cc27ca777..def4b3ce3 100644 --- a/job-client/src/main/java/com/lts/job/client/JobClient.java +++ b/job-client/src/main/java/com/lts/job/client/JobClient.java @@ -10,11 +10,13 @@ import com.lts.job.core.cluster.AbstractClientNode; import com.lts.job.core.constant.Constants; import com.lts.job.core.domain.Job; +import com.lts.job.core.exception.JobSubmitException; import com.lts.job.core.exception.JobTrackerNotFoundException; import com.lts.job.core.protocol.JobProtos; import com.lts.job.core.protocol.command.JobSubmitRequest; import com.lts.job.core.protocol.command.JobSubmitResponse; import com.lts.job.core.util.BatchUtils; +import com.lts.job.core.util.CollectionUtils; import com.lts.job.remoting.InvokeCallback; import com.lts.job.remoting.exception.RemotingCommandFieldCheckException; import com.lts.job.remoting.netty.NettyRequestProcessor; @@ -44,6 +46,16 @@ public JobClient() { config.setNodeGroup(Constants.DEFAULT_NODE_JOB_CLIENT_GROUP); } + @Override + protected void nodeEnable() { + // TODO + } + + @Override + protected void nodeDisable() { + // TODO + } + /** * @param job * @return @@ -53,6 +65,17 @@ public Response submitJob(Job job) { } protected Response _submitJob(final List jobs) { + // 参数验证 + if (CollectionUtils.isEmpty(jobs)) { + throw new JobSubmitException("提交任务不能为空!"); + } + for (Job job : jobs) { + if (job == null) { + throw new JobSubmitException("提交任务不能为空!"); + } else { + job.checkField(); + } + } final Response response = new Response(); diff --git a/job-core/src/main/java/com/lts/job/core/Application.java b/job-core/src/main/java/com/lts/job/core/Application.java index de84071d3..17e2fff46 100644 --- a/job-core/src/main/java/com/lts/job/core/Application.java +++ b/job-core/src/main/java/com/lts/job/core/Application.java @@ -1,8 +1,9 @@ package com.lts.job.core; import com.lts.job.core.cluster.MasterElector; -import com.lts.job.core.cluster.NodeManager; +import com.lts.job.core.cluster.SubscribedNodeManager; import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.ec.EventCenter; import com.lts.job.core.protocol.command.CommandBodyWrapper; import com.lts.job.core.registry.PathParser; @@ -14,20 +15,22 @@ public abstract class Application { public Application() { this.commandBodyWrapper = new CommandBodyWrapper(this); - this.nodeManager = new NodeManager(this); + this.subscribedNodeManager = new SubscribedNodeManager(this); this.masterElector = new MasterElector(this); } // 节点配置信息 private JobNodeConfig config; // 节点管理 - private NodeManager nodeManager; + private SubscribedNodeManager subscribedNodeManager; // master节点选举者 private MasterElector masterElector; // 节点通信CommandBody包装器 private CommandBodyWrapper commandBodyWrapper; // 节点路径解析器 private PathParser pathParser; + // 事件中心 + private EventCenter eventCenter; public PathParser getPathParser() { return pathParser; @@ -53,12 +56,12 @@ public void setConfig(JobNodeConfig config) { this.config = config; } - public NodeManager getNodeManager() { - return nodeManager; + public SubscribedNodeManager getSubscribedNodeManager() { + return subscribedNodeManager; } - public void setNodeManager(NodeManager nodeManager) { - this.nodeManager = nodeManager; + public void setSubscribedNodeManager(SubscribedNodeManager subscribedNodeManager) { + this.subscribedNodeManager = subscribedNodeManager; } public MasterElector getMasterElector() { @@ -69,4 +72,12 @@ public void setMasterElector(MasterElector masterElector) { this.masterElector = masterElector; } + public EventCenter getEventCenter() { + return eventCenter; + } + + public void setEventCenter(EventCenter eventCenter) { + this.eventCenter = eventCenter; + } + } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java index cdd1b8082..38e6cafca 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java @@ -1,10 +1,12 @@ package com.lts.job.core.cluster; import com.lts.job.core.Application; +import com.lts.job.core.constant.Constants; import com.lts.job.core.loadbalance.LoadBalance; import com.lts.job.core.loadbalance.RandomLoadBalance; import com.lts.job.core.remoting.HeartBeatMonitor; import com.lts.job.core.remoting.RemotingClientDelegate; +import com.lts.job.core.factory.NamedThreadFactory; import com.lts.job.core.util.StringUtils; import com.lts.job.remoting.netty.NettyClientConfig; import com.lts.job.remoting.netty.NettyRemotingClient; @@ -38,7 +40,8 @@ protected void innerStart() { if (defaultProcessor != null) { remotingClient.registerDefaultProcessor(defaultProcessor, - Executors.newCachedThreadPool()); + Executors.newFixedThreadPool(Constants.AVAILABLE_PROCESSOR * 2, + new NamedThreadFactory(AbstractClientNode.class.getSimpleName()))); } } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java index dfc640048..a2b4be9f9 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java @@ -1,12 +1,15 @@ package com.lts.job.core.cluster; import com.lts.job.core.Application; +import com.lts.job.core.constant.EcTopic; import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.ec.*; import com.lts.job.core.factory.JobNodeConfigFactory; import com.lts.job.core.factory.NodeFactory; import com.lts.job.core.listener.MasterNodeChangeListener; import com.lts.job.core.listener.MasterNodeElectionListener; import com.lts.job.core.listener.NodeChangeListener; +import com.lts.job.core.listener.SelfChangeListener; import com.lts.job.core.registry.Registry; import com.lts.job.core.registry.ZkNodeRegistry; import com.lts.job.core.util.GenericsUtils; @@ -30,9 +33,13 @@ public AbstractJobNode() { application = getApplication(); config = JobNodeConfigFactory.getDefaultConfig(); application.setConfig(config); + // 事件中心 + application.setEventCenter(new JvmEventCenter()); this.registry = new ZkNodeRegistry(application); // 用于master选举的监听器 addNodeChangeListener(new MasterNodeElectionListener(application)); + // 监听自己节点变化（如，当前节点被禁用了） + addNodeChangeListener(new SelfChangeListener(application)); } final public void start() { @@ -42,6 +49,20 @@ final public void start() { LOGGER.info("当前节点配置:{}", config); + // 监听节点 启用/禁用消息 + application.getEventCenter().subscribe( + new String[]{EcTopic.NODE_DISABLE, EcTopic.NODE_ENABLE}, + new EventSubscriber(node.getIdentity(), new Observer() { + @Override + public void onObserved(EventInfo eventInfo) { + if (EcTopic.NODE_DISABLE.equals(eventInfo.getTopic())) { + nodeDisable(); + } else { + nodeEnable(); + } + } + })); + innerStart(); registry.register(node); @@ -69,6 +90,10 @@ final public void stop() { protected abstract void innerStop(); + protected abstract void nodeEnable(); + + protected abstract void nodeDisable(); + @SuppressWarnings("unchecked") private App getApplication() { try { diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractServerNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractServerNode.java index 74a8b4941..cf28617bd 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractServerNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractServerNode.java @@ -1,7 +1,9 @@ package com.lts.job.core.cluster; import com.lts.job.core.Application; +import com.lts.job.core.constant.Constants; import com.lts.job.core.remoting.RemotingServerDelegate; +import com.lts.job.core.factory.NamedThreadFactory; import com.lts.job.remoting.netty.NettyRemotingServer; import com.lts.job.remoting.netty.NettyRequestProcessor; import com.lts.job.remoting.netty.NettyServerConfig; @@ -30,7 +32,7 @@ protected void innerStart() { if (defaultProcessor != null) { remotingServer.registerDefaultProcessor(defaultProcessor, - Executors.newCachedThreadPool()); + Executors.newFixedThreadPool(Constants.AVAILABLE_PROCESSOR * 2, new NamedThreadFactory(AbstractServerNode.class.getSimpleName()))); } } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java b/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java index a7b880675..79ad12228 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java @@ -70,7 +70,7 @@ public void removeNode(Node removedNode) { if (master != null) { if (master.getIdentity().equals(removedNode.getIdentity())) { // 如果挂掉的是master, 需要重新选举 - List nodes = application.getNodeManager().getNodeList(application.getConfig().getNodeType(), application.getConfig().getNodeGroup()); + List nodes = application.getSubscribedNodeManager().getNodeList(application.getConfig().getNodeType(), application.getConfig().getNodeGroup()); if (CollectionUtils.isNotEmpty(nodes)) { Node newMaster = null; for (Node node : nodes) { diff --git a/job-core/src/main/java/com/lts/job/core/cluster/NodeManager.java b/job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java similarity index 95% rename from job-core/src/main/java/com/lts/job/core/cluster/NodeManager.java rename to job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java index 3be57d1fe..d6906f5d1 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/NodeManager.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java @@ -15,14 +15,14 @@ * @author Robert HG (254963746@qq.com) on 6/22/14. * 节点管理 (主要用于管理自己关注的节点) */ -public class NodeManager { +public class SubscribedNodeManager { - private static final Logger LOGGER = LoggerFactory.getLogger(NodeManager.class); + private static final Logger LOGGER = LoggerFactory.getLogger(SubscribedNodeManager.class); private final ConcurrentHashMap> NODES = new ConcurrentHashMap>(); private Application application; - public NodeManager(Application application) { + public SubscribedNodeManager(Application application) { this.application = application; } diff --git a/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java b/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java new file mode 100644 index 000000000..b001ccdc2 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java @@ -0,0 +1,14 @@ +package com.lts.job.core.constant; + +/** + * Created by hugui on 5/11/15. + */ +public interface EcTopic { + + // 工作线程变化 + String WORK_THREAD_CHANGE = "WORK_THREAD_CHANGE"; + // 节点启用 + String NODE_ENABLE = "NODE_ENABLE"; + // 节点禁用 + String NODE_DISABLE = "NODE_DISABLE"; +} diff --git a/job-core/src/main/java/com/lts/job/core/domain/Job.java b/job-core/src/main/java/com/lts/job/core/domain/Job.java index ca7606bca..72bcca006 100644 --- a/job-core/src/main/java/com/lts/job/core/domain/Job.java +++ b/job-core/src/main/java/com/lts/job/core/domain/Job.java @@ -1,6 +1,7 @@ package com.lts.job.core.domain; +import com.lts.job.core.exception.JobSubmitException; import com.lts.job.core.util.JSONUtils; import com.lts.job.remoting.annotation.NotNull; @@ -18,9 +19,8 @@ public class Job { /** * 优先级 (数值越大 优先级越低) */ - protected Integer priority = 0; + protected Integer priority = 100; // 提交的节点 （可以手动指定） - @NotNull protected String submitNodeGroup; // 执行的节点 @NotNull @@ -136,4 +136,13 @@ public void setTriggerTime(Long triggerTime) { public String toString() { return JSONUtils.toJSONString(this); } + + public void checkField() throws JobSubmitException { + if(taskId == null){ + throw new JobSubmitException("taskId不能为空!"); + } + if(taskTrackerNodeGroup == null){ + throw new JobSubmitException("taskTrackerNodeGroup不能为空!"); + } + } } diff --git a/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java b/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java index 9b46944e7..1ccecc3b8 100644 --- a/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java +++ b/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java @@ -9,6 +9,8 @@ */ public class JobNodeConfig { + // 节点是否可用 + private boolean available = true; // 应用节点组 private String nodeGroup; // 唯一标识 @@ -97,13 +99,21 @@ public String getJobInfoSavePath() { } public void setJobInfoSavePath(String jobInfoSavePath) { - this.jobInfoSavePath = jobInfoSavePath + "/.lts"; + this.jobInfoSavePath = jobInfoSavePath + "/.lts"; } public String getFilePath() { return jobInfoSavePath + "/" + nodeType + "/" + nodeGroup + "/"; } + public boolean isAvailable() { + return available; + } + + public void setAvailable(boolean available) { + this.available = available; + } + @Override public String toString() { return JSONUtils.toJSONString(this); diff --git a/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java b/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java new file mode 100644 index 000000000..d750ca267 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java @@ -0,0 +1,27 @@ +package com.lts.job.core.exception; + +/** + * Created by hugui on 5/12/15. + */ +public class JobSubmitException extends RuntimeException { + + public JobSubmitException() { + super(); + } + + public JobSubmitException(String message) { + super(message); + } + + public JobSubmitException(String message, Throwable cause) { + super(message, cause); + } + + public JobSubmitException(Throwable cause) { + super(cause); + } + + protected JobSubmitException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/job-core/src/main/java/com/lts/job/registry/util/NamedThreadFactory.java b/job-core/src/main/java/com/lts/job/core/factory/NamedThreadFactory.java similarity index 94% rename from job-core/src/main/java/com/lts/job/registry/util/NamedThreadFactory.java rename to job-core/src/main/java/com/lts/job/core/factory/NamedThreadFactory.java index 5e5a065fa..79414222e 100755 --- a/job-core/src/main/java/com/lts/job/registry/util/NamedThreadFactory.java +++ b/job-core/src/main/java/com/lts/job/core/factory/NamedThreadFactory.java @@ -1,9 +1,10 @@ -package com.lts.job.registry.util; +package com.lts.job.core.factory; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; /** + * 带有名称的 线程 工厂类 * @author Robert HG (254963746@qq.com) on 5/5/14. */ public class NamedThreadFactory implements ThreadFactory { diff --git a/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java b/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java new file mode 100644 index 000000000..578c7be37 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java @@ -0,0 +1,56 @@ +package com.lts.job.core.listener; + +import com.lts.job.core.Application; +import com.lts.job.core.cluster.Node; +import com.lts.job.core.cluster.NodeType; +import com.lts.job.core.constant.EcTopic; +import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.ec.EventCenter; +import com.lts.job.ec.EventInfo; + +import java.util.List; + +/** + * 用来监听自己的节点信息变化 + * Created by hugui on 5/11/15. + */ +public class SelfChangeListener implements NodeChangeListener { + + private JobNodeConfig config; + private EventCenter eventCenter; + + public SelfChangeListener(Application application) { + this.config = application.getConfig(); + this.eventCenter = application.getEventCenter(); + } + + @Override + public void addNode(Node node) { + if (node.getIdentity().equals(config.getIdentity())) { + // 是当前节点, 看看节点配置是否发生变化 + // 1. 看 threads 有没有改变 , 目前只有 TASK_TRACKER 对 threads起作用 + if (node.getNodeType().equals(NodeType.TASK_TRACKER) + && (node.getThreads() != config.getWorkThreads())) { + config.setWorkThreads(node.getThreads()); + eventCenter.publishAsync(new EventInfo(EcTopic.WORK_THREAD_CHANGE)); + } + + // 2. 看 available 有没有改变 + if (node.isAvailable() != config.isAvailable()) { + String topic = node.isAvailable() ? EcTopic.NODE_ENABLE : EcTopic.NODE_DISABLE; + config.setAvailable(node.isAvailable()); + eventCenter.publishAsync(new EventInfo(topic)); + } + } + } + + @Override + public void removeNode(Node node) { + // do nothing + } + + @Override + public void addNodes(List nodes) { + // do nothing + } +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java index 53de44f96..eeb17b842 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java +++ b/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java @@ -90,7 +90,7 @@ protected void doSubscribe(Node node) { for (String child : children) { Node listenedNode = zkPathParser.parse(listenNodePath + "/" + child); listenedNodes.add(listenedNode); - application.getNodeManager().addNode(listenedNode); + application.getSubscribedNodeManager().addNode(listenedNode); } if (CollectionUtils.isNotEmpty(nodeChangeListeners)) { for (NodeChangeListener nodeChangeListener : nodeChangeListeners) { @@ -111,7 +111,7 @@ protected void doUnSubscribe(Node node) { for (NodeType nodeType : listenNodeTypes) { zkClient.removeChildListener(zkPathParser.getPath(nodeType), listener); - application.getNodeManager().destroy(); + application.getSubscribedNodeManager().destroy(); } } } @@ -146,7 +146,7 @@ public void childChanged(String path, List children) { for (String child : addChildren) { Node node = zkPathParser.parse(path + "/" + child); - application.getNodeManager().addNode(node); + application.getSubscribedNodeManager().addNode(node); if (CollectionUtils.isNotEmpty(nodeChangeListeners)) { for (NodeChangeListener nodeChangeListener : nodeChangeListeners) { nodeChangeListener.addNode(node); @@ -158,7 +158,7 @@ public void childChanged(String path, List children) { if (CollectionUtils.isNotEmpty(decChildren)) { for (String child : decChildren) { Node node = zkPathParser.parse(path + "/" + child); - application.getNodeManager().removeNode(node); + application.getSubscribedNodeManager().removeNode(node); if (CollectionUtils.isNotEmpty(nodeChangeListeners)) { for (NodeChangeListener nodeChangeListener : nodeChangeListeners) { nodeChangeListener.removeNode(node); diff --git a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java b/job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java new file mode 100644 index 000000000..a49a11917 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java @@ -0,0 +1,114 @@ +package com.lts.job.core.registry; + +import com.lts.job.core.cluster.Node; +import com.lts.job.core.cluster.NodeType; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author Robert HG (254963746@qq.com) on 5/11/15. + *

+ * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER + *

+ */ +public class ZkNodeUtils { + + public static String getBasePath(String clusterName) { + return "/LTS/" + clusterName + "/NODES"; + } + + public static Node parse(String clusterName, String fullPath) { + Node node = new Node(); + + node.setPath(fullPath); + String nodeType = getMatcher(getBasePath(clusterName) + "/(.*)/", fullPath); + node.setNodeType(NodeType.valueOf(nodeType)); + + String url = getMatcher(getBasePath(clusterName) + "/" + nodeType + "/" + nodeType + ":\\\\\\\\(.*)", fullPath); + + String address = url.split("\\?")[0]; + String ip = address.split(":")[0]; + + node.setIp(ip); + if (address.contains(":")) { + String port = address.split(":")[1]; + if (port != null && !"".equals(port.trim())) { + node.setPort(Integer.valueOf(port)); + } + } + String params = url.split("\\?")[1]; + + String[] paramArr = params.split("&"); + for (String paramEntry : paramArr) { + String key = paramEntry.split("=")[0]; + String value = paramEntry.split("=")[1]; + + if ("group".equals(key)) { + node.setGroup(value); + } else if ("threads".equals(key)) { + node.setThreads(Integer.valueOf(value)); + } else if ("identity".equals(key)) { + node.setIdentity(value); + } else if ("createTime".equals(key)) { + node.setCreateTime(Long.valueOf(value)); + } else if ("isAvailable".equals(key)) { + node.setAvailable(Boolean.valueOf(value)); + } else if ("listenNodeTypes".equals(key)) { + String[] nodeTypes = value.split(","); + for (String type : nodeTypes) { + node.addListenNodeType(NodeType.valueOf(type)); + } + } + } + return node; + } + + public static String getPath(String clusterName, Node node) { + StringBuilder path = new StringBuilder(); + path.append(getBasePath(clusterName)) + .append("/") + .append(node.getNodeType()) + .append("/") + .append(node.getNodeType()) + .append(":\\\\") + .append(node.getIp()); + + if (node.getPort() != null && node.getPort() != 0) { + path.append(":").append(node.getPort()); + } + + path.append("?") + .append("group=") + .append(node.getGroup()); + if (node.getThreads() != 0) { + path.append("&threads=") + .append(node.getThreads()); + } + + path.append("&identity=") + .append(node.getIdentity()) + .append("&createTime=") + .append(node.getCreateTime()) + .append("&isAvailable=") + .append(node.isAvailable()); + + if (node.getListenNodeTypes() != null && node.getListenNodeTypes().size() != 0) { + path.append("&listenNodeTypes="); + for (NodeType nodeType : node.getListenNodeTypes()) { + path.append(nodeType).append(","); + } + path.deleteCharAt(path.length() - 1); + } + return path.toString(); + } + + private static String getMatcher(String regex, String source) { + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(source); + while (matcher.find()) { + return matcher.group(1);//只取第一组 + } + return ""; + } +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java b/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java index cf5ad55c3..b9eb8a4af 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java +++ b/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java @@ -1,22 +1,18 @@ package com.lts.job.core.registry; - import com.lts.job.core.Application; import com.lts.job.core.cluster.Node; import com.lts.job.core.cluster.NodeType; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - /** * @author Robert HG (254963746@qq.com) on 6/23/14. - *

- * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER - *

- * 节点 zookeeper上路径解析工具 + *

+ * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER + *

+ * 节点 zookeeper上路径解析工具 */ -public class ZkPathParser implements PathParser{ +public class ZkPathParser implements PathParser { private Application application; @@ -24,111 +20,16 @@ public ZkPathParser(Application application) { this.application = application; } - public String getBasePath(){ - // 集群名字 - return "/LTS/" + application.getConfig().getClusterName() + "/NODES"; - } - public Node parse(String fullPath) { - Node node = new Node(); - - node.setPath(fullPath); - String nodeType = getMatcher(getBasePath() + "/(.*)/", fullPath); - node.setNodeType(NodeType.valueOf(nodeType)); - - String url = getMatcher(getBasePath() + "/" + nodeType + "/" + nodeType + ":\\\\\\\\(.*)", fullPath); - - String address = url.split("\\?")[0]; - String ip = address.split(":")[0]; - - node.setIp(ip); - if(address.contains(":")){ - String port = address.split(":")[1]; - if (port != null && !"".equals(port.trim())) { - node.setPort(Integer.valueOf(port)); - } - } - String params = url.split("\\?")[1]; - - String[] paramArr = params.split("&"); - for (String paramEntry : paramArr) { - String key = paramEntry.split("=")[0]; - String value = paramEntry.split("=")[1]; - - if ("group".equals(key)) { - node.setGroup(value); - } else if ("threads".equals(key)) { - node.setThreads(Integer.valueOf(value)); - } else if ("identity".equals(key)) { - node.setIdentity(value); - } else if ("createTime".equals(key)) { - node.setCreateTime(Long.valueOf(value)); - } else if ("isAvailable".equals(key)) { - node.setAvailable(Boolean.valueOf(value)); - } else if ("listenNodeTypes".equals(key)) { - String[] nodeTypes = value.split(","); - for (String type : nodeTypes) { - node.addListenNodeType(NodeType.valueOf(type)); - } - } - - } - - return node; + return ZkNodeUtils.parse(application.getConfig().getClusterName(), fullPath); } public String getPath(Node node) { - StringBuilder path = new StringBuilder(); - path.append(getBasePath()) - .append("/") - .append(node.getNodeType()) - .append("/") - .append(node.getNodeType()) - .append(":\\\\") - .append(node.getIp()); - - if (node.getPort() != null && node.getPort() != 0) { - path.append(":").append(node.getPort()); - } - - path.append("?") - .append("group=") - .append(node.getGroup()); - if (node.getThreads() != 0) { - path.append("&threads=") - .append(node.getThreads()); - } - - path.append("&identity=") - .append(node.getIdentity()) - .append("&createTime=") - .append(node.getCreateTime()) - .append("&isAvailable=") - .append(node.isAvailable()); - - if (node.getListenNodeTypes() != null && node.getListenNodeTypes().size() != 0) { - path.append("&listenNodeTypes="); - for (NodeType nodeType : node.getListenNodeTypes()) { - path.append(nodeType).append(","); - } - path.deleteCharAt(path.length() - 1); - } - - - return path.toString(); + return ZkNodeUtils.getPath(application.getConfig().getClusterName(), node); } public String getPath(NodeType nodeType) { - return getBasePath() + "/" + nodeType; - } - - private String getMatcher(String regex, String source) { - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(source); - while (matcher.find()) { - return matcher.group(1);//只取第一组 - } - return ""; + return ZkNodeUtils.getBasePath(application.getConfig().getClusterName()) + "/" + nodeType; } } diff --git a/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java b/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java index cd8c177bc..551a05925 100644 --- a/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java +++ b/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java @@ -51,7 +51,7 @@ private class HeartBeat implements Runnable { @Override public void run() { try { - List jobTrackers = application.getNodeManager().getNodeList(NodeType.JOB_TRACKER); + List jobTrackers = application.getSubscribedNodeManager().getNodeList(NodeType.JOB_TRACKER); if (jobTrackers == null) { return; } diff --git a/job-core/src/main/java/com/lts/job/ec/EventCenter.java b/job-core/src/main/java/com/lts/job/ec/EventCenter.java new file mode 100644 index 000000000..2870408f2 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/ec/EventCenter.java @@ -0,0 +1,47 @@ +package com.lts.job.ec; + +/** + * 事件中心接口 + * Created by hugui on 5/11/15. + */ +public interface EventCenter { + + /** + * 订阅主题 + * + * @param topic + * @param subscriber + */ + public void subscribe(String topic, EventSubscriber subscriber); + + /** + * 订阅主题 + * + * @param topics + * @param subscriber + */ + public void subscribe(String[] topics, EventSubscriber subscriber); + + /** + * 取消订阅主题 + * + * @param topic + * @param subscriber + */ + public void unSubscribe(String topic, EventSubscriber subscriber); + + /** + * 同步发布主题消息 + * + * @param eventInfo + */ + public void publishSync(EventInfo eventInfo); + + /** + * 异步发送主题消息 + * + * @param eventInfo + */ + public void publishAsync(EventInfo eventInfo); + +} diff --git a/job-core/src/main/java/com/lts/job/ec/EventInfo.java b/job-core/src/main/java/com/lts/job/ec/EventInfo.java new file mode 100644 index 000000000..baf04d038 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/ec/EventInfo.java @@ -0,0 +1,51 @@ +package com.lts.job.ec; + +import java.util.HashMap; +import java.util.Map; + +/** + * 事件信息 + * Created by hugui on 5/11/15. + */ +public class EventInfo { + + private String topic; + private Map params; + + public EventInfo(String topic) { + this.topic = topic; + } + + public void setParam(String key, Object value) { + if (params == null) { + params = new HashMap(); + } + params.put(key, value); + } + + public Object removeParam(String key) { + if (params != null) { + return params.remove(key); + } + return null; + } + + public Object getParam(String key) { + if (params != null) { + return params.remove(key); + } + return null; + } + + public Map getParams() { + return params == null ? new HashMap() : params; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java b/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java new file mode 100644 index 000000000..223e3151f --- /dev/null +++ b/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java @@ -0,0 +1,33 @@ +package com.lts.job.ec; + +/** + * 事件订阅者 + * Created by hugui on 5/11/15. + */ +public class EventSubscriber { + + public EventSubscriber(String id, Observer observer) { + this.id = id; + this.observer = observer; + } + + private String id; + + private Observer observer; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Observer getObserver() { + return observer; + } + + public void setObserver(Observer observer) { + this.observer = observer; + } +} diff --git a/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java b/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java new file mode 100644 index 000000000..3d21d845c --- /dev/null +++ b/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java @@ -0,0 +1,85 @@ +package com.lts.job.ec; + +import com.lts.job.core.constant.Constants; +import com.lts.job.core.util.ConcurrentHashSet; +import com.lts.job.core.util.JSONUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * 在一个jvm中的pub sub 简易实现 + * Created by hugui on 5/12/15. + */ +public class JvmEventCenter implements EventCenter{ + + private static final Logger LOGGER = LoggerFactory.getLogger(EventCenter.class.getName()); + + private Map> ecMap = + new ConcurrentHashMap>(); + + private ExecutorService executor = Executors.newFixedThreadPool(Constants.AVAILABLE_PROCESSOR * 2); + + public void subscribe(String topic, EventSubscriber subscriber) { + Set subscribers = ecMap.get(topic); + if (subscribers == null) { + synchronized (ecMap) { + subscribers = new ConcurrentHashSet(); + ecMap.put(topic, subscribers); + } + } + subscribers.add(subscriber); + } + + public void subscribe(String[] topics, EventSubscriber subscriber) { + for (String topic : topics) { + subscribe(topic, subscriber); + } + } + + public void unSubscribe(String topic, EventSubscriber subscriber) { + Set subscribers = ecMap.get(topic); + if (subscribers != null) { + for (EventSubscriber eventSubscriber : subscribers) { + if (eventSubscriber.getId().equals(subscriber.getId())) { + subscribers.remove(eventSubscriber); + } + } + } + } + + public void publishSync(EventInfo eventInfo) { + Set subscribers = ecMap.get(eventInfo.getTopic()); + if (subscribers != null) { + for (EventSubscriber subscriber : subscribers) { + eventInfo.setTopic(eventInfo.getTopic()); + subscriber.getObserver().onObserved(eventInfo); + } + } + } + + public void publishAsync(final EventInfo eventInfo) { + executor.submit(new Runnable() { + @Override + public void run() { + String topic = eventInfo.getTopic(); + try { + Set subscribers = ecMap.get(topic); + if (subscribers != null) { + for (EventSubscriber subscriber : subscribers) { + eventInfo.setTopic(topic); + subscriber.getObserver().onObserved(eventInfo); + } + } + } catch (Throwable t) { + LOGGER.error("topic:{}, eventInfo:{}", topic, JSONUtils.toJSONString(eventInfo), t); + } + } + }); + } +} diff --git a/job-core/src/main/java/com/lts/job/ec/Observer.java b/job-core/src/main/java/com/lts/job/ec/Observer.java new file mode 100644 index 000000000..4d98c5213 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/ec/Observer.java @@ -0,0 +1,11 @@ +package com.lts.job.ec; + +/** + * 事件观察者接口 + * Created by hugui on 5/11/15. + */ +public interface Observer { + + public void onObserved(EventInfo eventInfo); + +} diff --git a/job-task-tracker/src/main/java/com/lts/job/task/tracker/TaskTracker.java b/job-task-tracker/src/main/java/com/lts/job/task/tracker/TaskTracker.java index 5c75fc9cc..aa7fdff4b 100644 --- a/job-task-tracker/src/main/java/com/lts/job/task/tracker/TaskTracker.java +++ b/job-task-tracker/src/main/java/com/lts/job/task/tracker/TaskTracker.java @@ -2,7 +2,11 @@ import com.lts.job.core.cluster.AbstractClientNode; import com.lts.job.core.constant.Constants; +import com.lts.job.core.constant.EcTopic; import com.lts.job.core.constant.Level; +import com.lts.job.ec.EventInfo; +import com.lts.job.ec.EventSubscriber; +import com.lts.job.ec.Observer; import com.lts.job.remoting.netty.NettyRequestProcessor; import com.lts.job.task.tracker.domain.TaskTrackerApplication; import com.lts.job.task.tracker.domain.TaskTrackerNode; @@ -26,6 +30,18 @@ public TaskTracker() { @Override protected void innerStart() { + // 向事件中心注册事件 + application.getEventCenter().subscribe( + EcTopic.WORK_THREAD_CHANGE, + new EventSubscriber(node.getIdentity(), new Observer() { + @Override + public void onObserved(EventInfo eventInfo) { + // 改变工作线程大小 + int threads = config.getWorkThreads(); + application.getRunnerPool().setMaximumPoolSize(threads); + } + })); + // 设置 线程池 application.setRunnerPool(new RunnerPool(application)); super.innerStart(); application.setRemotingClient(remotingClient); @@ -39,6 +55,16 @@ protected void innerStop() { jobPullMachine.stop(); } + @Override + protected void nodeEnable() { + // TODO + } + + @Override + protected void nodeDisable() { + // TODO + } + @Override protected NettyRequestProcessor getDefaultProcessor() { return new RemotingDispatcher(remotingClient, application); diff --git a/job-task-tracker/src/main/java/com/lts/job/task/tracker/runner/RunnerPool.java b/job-task-tracker/src/main/java/com/lts/job/task/tracker/runner/RunnerPool.java index 439c44017..025ba4467 100644 --- a/job-task-tracker/src/main/java/com/lts/job/task/tracker/runner/RunnerPool.java +++ b/job-task-tracker/src/main/java/com/lts/job/task/tracker/runner/RunnerPool.java @@ -77,6 +77,20 @@ public int getAvailablePoolSize() { return threadPoolExecutor.getMaximumPoolSize() - threadPoolExecutor.getActiveCount(); } + public void setMaximumPoolSize(int maximumPoolSize) { + if (maximumPoolSize == 0) { + throw new IllegalArgumentException("maximumPoolSize不能为0!"); + } + + int corePollSize = threadPoolExecutor.getCorePoolSize(); + if (maximumPoolSize < corePollSize) { + threadPoolExecutor.setCorePoolSize(maximumPoolSize); + } + threadPoolExecutor.setMaximumPoolSize(maximumPoolSize); + + LOGGER.info("maximumPoolSize更新为{}", maximumPoolSize); + } + /** * 得到最大线程数 * diff --git a/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java b/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java new file mode 100644 index 000000000..20881646f --- /dev/null +++ b/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java @@ -0,0 +1,63 @@ +package com.lts.job.task.tracker; + +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Created by hugui on 5/11/15. + */ +public class ThreadPoolDynamicTest { + + public static void main(String[] args) { + + ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(4, 8, 30, TimeUnit.SECONDS, + new SynchronousQueue(), // 直接提交给线程而不保持它们 + new ThreadPoolExecutor.AbortPolicy()); // A handler for rejected tasks that throws a + + + for (int i = 1; i < 12; i++) { + try { + final int finalI = i; + threadPoolExecutor.submit(new Runnable() { + @Override + public void run() { + try { + System.out.println("第" + finalI + "个开始执行"); + Thread.sleep(1000000000L); + } catch (InterruptedException e) { + System.out.println("第" + finalI + "个执行结果失败"); + } + } + }); + System.out.println("第" + i + "个提交成功"); + } catch (Exception e) { + System.out.println("第" + i + "个提交失败"); + } + } + + threadPoolExecutor.setMaximumPoolSize(20); + + for (int i = 12; i < 20; i++) { + try { + final int finalI = i; + threadPoolExecutor.submit(new Runnable() { + @Override + public void run() { + try { + System.out.println("第" + finalI + "个开始执行"); + Thread.sleep(1000000000L); + } catch (InterruptedException e) { + System.out.println("第" + finalI + "个执行结果失败"); + } + } + }); + System.out.println("第" + i + "个提交成功"); + } catch (Exception e) { + System.out.println("第" + i + "个提交失败"); + } + } + + } + +} diff --git a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java index 12080a2bf..7184f84e5 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java @@ -58,6 +58,17 @@ protected void innerStart() { application.setRemotingServer(remotingServer); } + @Override + protected void nodeEnable() { + // TODO + + } + + @Override + protected void nodeDisable() { + // TODO 节点被禁用 + } + @Override protected NettyRequestProcessor getDefaultProcessor() { return new RemotingDispatcher(remotingServer, application); diff --git a/job-tracker/src/main/java/com/lts/job/tracker/processor/JobPullProcessor.java b/job-tracker/src/main/java/com/lts/job/tracker/processor/JobPullProcessor.java index 25e1d133c..9866afe95 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/processor/JobPullProcessor.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/processor/JobPullProcessor.java @@ -1,8 +1,10 @@ package com.lts.job.tracker.processor; +import com.lts.job.core.constant.Constants; import com.lts.job.core.protocol.JobProtos; import com.lts.job.core.protocol.command.JobPullRequest; import com.lts.job.core.remoting.RemotingServerDelegate; +import com.lts.job.core.factory.NamedThreadFactory; import com.lts.job.remoting.exception.RemotingCommandException; import com.lts.job.remoting.protocol.RemotingCommand; import com.lts.job.tracker.domain.JobTrackerApplication; @@ -28,7 +30,8 @@ public class JobPullProcessor extends AbstractProcessor { public JobPullProcessor(RemotingServerDelegate remotingServer, JobTrackerApplication application) { super(remotingServer, application); - executor = Executors.newCachedThreadPool(); + executor = Executors.newFixedThreadPool(Constants.AVAILABLE_PROCESSOR * 2 + , new NamedThreadFactory(JobPullProcessor.class.getSimpleName())); jobDistributor = new JobDistributor(application); } diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/checker/DeadJobChecker.java b/job-tracker/src/main/java/com/lts/job/tracker/support/checker/DeadJobChecker.java index 65e893096..f0d86cc91 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/checker/DeadJobChecker.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/checker/DeadJobChecker.java @@ -81,7 +81,7 @@ public void run() { // 一般来说这个是没有多大的，我就不分页去查询了 List jobPos = jobQueue.getByLimitExecTime(MAX_DEAD_CHECK_TIME); if (jobPos != null && jobPos.size() > 0) { - List nodes = application.getNodeManager().getNodeList(NodeType.TASK_TRACKER); + List nodes = application.getSubscribedNodeManager().getNodeList(NodeType.TASK_TRACKER); HashSet identities = new HashSet(); if (CollectionUtils.isNotEmpty(nodes)) { for (Node node : nodes) { diff --git a/pom.xml b/pom.xml index a283f4aba..9daa5de0e 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ job-client job-example job-extensions - + job-admin From 22b71df174f92c972acf1a4aa4f3c01166e14427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E8=B4=B5?= Date: Mon, 18 May 2015 00:44:53 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=8F=82=E8=80=83=20dubbo=20=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E5=A4=9A=E6=B3=A8=E5=86=8C=E4=B8=AD=E5=BF=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 24 +- .../web/controller/AbstractController.java | 2 +- .../job/web/controller/NodeController.java | 2 +- .../lts/job/web/support/AppConfigurer.java | 2 +- .../lts/job/web/support/node/NodeManager.java | 2 +- .../job/web/support/node/ZkNodeManager.java | 16 +- job-core/pom.xml | 8 - .../java/com/lts/job/core/Application.java | 19 +- .../lts/job/core/cluster/AbstractJobNode.java | 112 ++++-- .../com/lts/job/core/cluster/JobNode.java | 4 + .../lts/job/core/cluster/MasterElector.java | 42 +-- .../java/com/lts/job/core/cluster/Node.java | 20 +- .../core/cluster/SubscribedNodeManager.java | 87 +++-- .../com/lts/job/core/constant/Constants.java | 24 ++ .../com/lts/job/core/constant/EcTopic.java | 2 +- .../lts/job/core/domain/JobNodeConfig.java | 33 +- .../core/exception/JobSubmitException.java | 2 +- .../core/exception/NodeRegistryException.java | 27 ++ .../core/factory/JobNodeConfigFactory.java | 2 +- .../com/lts/job/core/factory/NodeFactory.java | 4 +- ...istener.java => MasterChangeListener.java} | 2 +- ...tener.java => MasterElectionListener.java} | 39 ++- .../job/core/listener/NodeChangeListener.java | 19 +- .../job/core/listener/SelfChangeListener.java | 21 +- .../job/core/registry/AbstractRegistry.java | 189 ++++++++++ .../job/core/registry/FailbackRegistry.java | 325 ++++++++++++++++++ ...kNodeUtils.java => NodeRegistryUtils.java} | 24 +- .../lts/job/core/registry/NotifyEvent.java | 10 + .../lts/job/core/registry/NotifyListener.java | 14 + .../com/lts/job/core/registry/PathParser.java | 14 - .../com/lts/job/core/registry/Registry.java | 20 +- .../job/core/registry/RegistryFactory.java | 26 ++ .../lts/job/core/registry/ZkNodeRegistry.java | 173 ---------- .../lts/job/core/registry/ZkPathParser.java | 35 -- .../core/registry/redis/RedisRegistry.java | 188 ++++++++++ .../registry/zookeeper/ZookeeperRegistry.java | 174 ++++++++++ .../job/core/remoting/HeartBeatMonitor.java | 3 +- .../main/java/com/lts/job/ec/EventCenter.java | 2 +- .../main/java/com/lts/job/ec/EventInfo.java | 2 +- .../java/com/lts/job/ec/EventSubscriber.java | 2 +- .../java/com/lts/job/ec/JvmEventCenter.java | 2 +- .../main/java/com/lts/job/ec/Observer.java | 2 +- .../job/remoting/netty/NettyClientConfig.java | 4 +- .../zookeeper/ChildListener.java | 2 +- .../zookeeper/StateListener.java | 2 +- .../zookeeper/ZookeeperClient.java | 6 +- .../curator/CuratorZookeeperClient.java | 237 +++++++++++++ .../serializer/SerializableSerializer.java | 40 +++ .../serializer/ZkMarshallingException.java | 27 ++ .../zookeeper/serializer/ZkSerializer.java | 11 + .../support/AbstractZookeeperClient.java | 8 +- .../zkclient/ZkClientZookeeperClient.java | 19 +- .../com/lts/job/core/FileAccessorTest.java | 55 --- .../java/com/lts/job/registry/ZooTest.java | 6 +- job-example/pom.xml | 5 + .../lts/job/example/api/JobClientTest.java | 6 +- .../lts/job/example/api/JobTrackerTest.java | 6 +- .../lts/job/example/api/TaskTrackerTest.java | 6 +- ...mpl.java => MasterChangeListenerImpl.java} | 4 +- .../main/resources/lts-spring-job-client.xml | 6 +- .../main/resources/lts-spring-job-tacker.xml | 6 +- .../resources/lts-spring-job-tasktracker.xml | 6 +- .../lts/job/spring/JobClientFactoryBean.java | 24 +- .../lts/job/spring/JobTrackerFactoryBean.java | 24 +- .../job/spring/TaskTrackerFactoryBean.java | 24 +- .../task/tracker/ThreadPoolDynamicTest.java | 2 +- .../java/com/lts/job/tracker/JobTracker.java | 23 +- .../job/tracker/channel/ChannelManager.java | 48 ++- .../job/tracker/logger/domain/BizLogPo.java | 2 +- .../tracker/support/TaskTrackerManager.java | 20 +- .../listener/JobNodeChangeListener.java | 43 ++- .../JobTrackerMasterChangeListener.java | 4 +- .../support/policy/OldDataDeletePolicy.java | 2 +- pom.xml | 33 +- 74 files changed, 1807 insertions(+), 624 deletions(-) create mode 100644 job-core/src/main/java/com/lts/job/core/exception/NodeRegistryException.java rename job-core/src/main/java/com/lts/job/core/listener/{MasterNodeChangeListener.java => MasterChangeListener.java} (89%) rename job-core/src/main/java/com/lts/job/core/listener/{MasterNodeElectionListener.java => MasterElectionListener.java} (52%) create mode 100644 job-core/src/main/java/com/lts/job/core/registry/AbstractRegistry.java create mode 100644 job-core/src/main/java/com/lts/job/core/registry/FailbackRegistry.java rename job-core/src/main/java/com/lts/job/core/registry/{ZkNodeUtils.java => NodeRegistryUtils.java} (81%) create mode 100644 job-core/src/main/java/com/lts/job/core/registry/NotifyEvent.java create mode 100644 job-core/src/main/java/com/lts/job/core/registry/NotifyListener.java delete mode 100644 job-core/src/main/java/com/lts/job/core/registry/PathParser.java create mode 100644 job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java delete mode 100644 job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java delete mode 100644 job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java create mode 100644 job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java create mode 100644 job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java rename job-core/src/main/java/com/lts/job/{registry => }/zookeeper/ChildListener.java (81%) rename job-core/src/main/java/com/lts/job/{registry => }/zookeeper/StateListener.java (84%) rename job-core/src/main/java/com/lts/job/{registry => }/zookeeper/ZookeeperClient.java (86%) create mode 100644 job-core/src/main/java/com/lts/job/zookeeper/curator/CuratorZookeeperClient.java create mode 100644 job-core/src/main/java/com/lts/job/zookeeper/serializer/SerializableSerializer.java create mode 100644 job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkMarshallingException.java create mode 100644 job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkSerializer.java rename job-core/src/main/java/com/lts/job/{registry => }/zookeeper/support/AbstractZookeeperClient.java (95%) rename job-core/src/main/java/com/lts/job/{registry => }/zookeeper/zkclient/ZkClientZookeeperClient.java (91%) delete mode 100644 job-core/src/test/java/com/lts/job/core/FileAccessorTest.java rename job-example/src/main/java/com/lts/job/example/support/{MasterNodeChangeListenerImpl.java => MasterChangeListenerImpl.java} (85%) diff --git a/README.md b/README.md index 7b8dad460..d850f7f4a 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) ```java final JobTracker jobTracker = new JobTracker(); // 节点信息配置 - jobTracker.setZookeeperAddress("localhost:2181"); + jobTracker.setRegistryAddress("zookeeper://localhost:2181"); // jobTracker.setListenPort(35001); // 默认 35001 // jobTracker.setClusterName("lts"); @@ -93,11 +93,11 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - + - + - + @@ -108,7 +108,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) TaskTracker taskTracker = new TaskTracker(); taskTracker.setJobRunnerClass(TestJobRunner.class); // jobClient.setClusterName("lts"); - taskTracker.setZookeeperAddress("localhost:2181"); + taskTracker.setRegistryAddress("zookeeper://localhost:2181"); taskTracker.setNodeGroup("test_trade_TaskTracker"); taskTracker.setWorkThreads(20); taskTracker.start(); @@ -136,12 +136,12 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - + - + - + @@ -153,7 +153,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) // JobClient jobClient = new JobClient(); jobClient.setNodeGroup("test_JobClient"); // jobClient.setClusterName("lts"); - jobClient.setZookeeperAddress("localhost:2181"); + jobClient.setRegistryAddress("zookeeper://localhost:2181"); jobClient.start(); // 提交任务 @@ -170,13 +170,13 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - + - + - + diff --git a/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java b/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java index 0778ac593..50551e873 100644 --- a/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java +++ b/job-admin/src/main/java/com/lts/job/web/controller/AbstractController.java @@ -1,7 +1,7 @@ package com.lts.job.web.controller; /** - * Created by hugui on 5/9/15. + * @author Robert HG (254963746@qq.com) on 5/9/15. */ public class AbstractController { diff --git a/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java b/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java index 3169d3182..d87101a0b 100644 --- a/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java +++ b/job-admin/src/main/java/com/lts/job/web/controller/NodeController.java @@ -3,7 +3,7 @@ import org.springframework.stereotype.Controller; /** - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ @Controller public class NodeController extends AbstractController{ diff --git a/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java b/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java index 0084d3101..4378c1ace 100644 --- a/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java +++ b/job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java @@ -3,7 +3,7 @@ import java.util.*; /** - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public class AppConfigurer { diff --git a/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java b/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java index 4f8616a50..3c62c07b1 100644 --- a/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java +++ b/job-admin/src/main/java/com/lts/job/web/support/node/NodeManager.java @@ -7,7 +7,7 @@ import java.util.Map; /** - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public interface NodeManager { diff --git a/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java b/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java index fdb94bb64..145279e1b 100644 --- a/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java +++ b/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java @@ -2,10 +2,10 @@ import com.lts.job.core.cluster.Node; import com.lts.job.core.cluster.NodeType; -import com.lts.job.core.registry.ZkNodeUtils; +import com.lts.job.core.registry.NodeRegistryUtils; import com.lts.job.core.util.CollectionUtils; -import com.lts.job.registry.zookeeper.ZookeeperClient; -import com.lts.job.registry.zookeeper.zkclient.ZkClientZookeeperClient; +import com.lts.job.zookeeper.ZookeeperClient; +import com.lts.job.zookeeper.zkclient.ZkClientZookeeperClient; import com.lts.job.web.support.AppConfigurer; import java.util.ArrayList; @@ -14,7 +14,7 @@ import java.util.Map; /** - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public class ZkNodeManager implements NodeManager { @@ -26,7 +26,7 @@ public ZkNodeManager() { @Override public Map> getAllNodes(String clusterName) { - String basePath = ZkNodeUtils.getBasePath(clusterName); + String basePath = NodeRegistryUtils.getRootPath(clusterName); List nodeTypes = zkClient.getChildren(basePath); Map> nodeMap = new HashMap>(); if (CollectionUtils.isNotEmpty(nodeTypes)) { @@ -35,7 +35,7 @@ public Map> getAllNodes(String clusterName) { if (CollectionUtils.isEmpty(nodes)) { List nodeList = new ArrayList(nodes.size()); for (String node : nodes) { - nodeList.add(ZkNodeUtils.parse(clusterName, node)); + nodeList.add(NodeRegistryUtils.parse(clusterName, node)); } nodeMap.put(NodeType.valueOf(nodeType), nodeList); } @@ -59,4 +59,8 @@ public void updateWorkThreads(Node node) { } + public static void main(String[] args) { + + + } } diff --git a/job-core/pom.xml b/job-core/pom.xml index bf51c1600..fba1b8c8c 100644 --- a/job-core/pom.xml +++ b/job-core/pom.xml @@ -10,12 +10,4 @@ 4.0.0 job-core - - - - org.fusesource.leveldbjni - leveldbjni-all - 1.8 - - \ No newline at end of file diff --git a/job-core/src/main/java/com/lts/job/core/Application.java b/job-core/src/main/java/com/lts/job/core/Application.java index 17e2fff46..0136a7430 100644 --- a/job-core/src/main/java/com/lts/job/core/Application.java +++ b/job-core/src/main/java/com/lts/job/core/Application.java @@ -3,9 +3,8 @@ import com.lts.job.core.cluster.MasterElector; import com.lts.job.core.cluster.SubscribedNodeManager; import com.lts.job.core.domain.JobNodeConfig; -import com.lts.job.ec.EventCenter; import com.lts.job.core.protocol.command.CommandBodyWrapper; -import com.lts.job.core.registry.PathParser; +import com.lts.job.ec.EventCenter; /** * @author Robert HG (254963746@qq.com) on 8/17/14. @@ -13,12 +12,6 @@ */ public abstract class Application { - public Application() { - this.commandBodyWrapper = new CommandBodyWrapper(this); - this.subscribedNodeManager = new SubscribedNodeManager(this); - this.masterElector = new MasterElector(this); - } - // 节点配置信息 private JobNodeConfig config; // 节点管理 @@ -27,19 +20,9 @@ public Application() { private MasterElector masterElector; // 节点通信CommandBody包装器 private CommandBodyWrapper commandBodyWrapper; - // 节点路径解析器 - private PathParser pathParser; // 事件中心 private EventCenter eventCenter; - public PathParser getPathParser() { - return pathParser; - } - - public void setPathParser(PathParser pathParser) { - this.pathParser = pathParser; - } - public CommandBodyWrapper getCommandBodyWrapper() { return commandBodyWrapper; } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java index a2b4be9f9..dae7d92a2 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java @@ -3,19 +3,26 @@ import com.lts.job.core.Application; import com.lts.job.core.constant.EcTopic; import com.lts.job.core.domain.JobNodeConfig; -import com.lts.job.ec.*; import com.lts.job.core.factory.JobNodeConfigFactory; import com.lts.job.core.factory.NodeFactory; -import com.lts.job.core.listener.MasterNodeChangeListener; -import com.lts.job.core.listener.MasterNodeElectionListener; +import com.lts.job.core.listener.MasterChangeListener; +import com.lts.job.core.listener.MasterElectionListener; import com.lts.job.core.listener.NodeChangeListener; import com.lts.job.core.listener.SelfChangeListener; -import com.lts.job.core.registry.Registry; -import com.lts.job.core.registry.ZkNodeRegistry; +import com.lts.job.core.protocol.command.CommandBodyWrapper; +import com.lts.job.core.registry.*; +import com.lts.job.core.util.CollectionUtils; import com.lts.job.core.util.GenericsUtils; +import com.lts.job.ec.EventInfo; +import com.lts.job.ec.EventSubscriber; +import com.lts.job.ec.JvmEventCenter; +import com.lts.job.ec.Observer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + /** * @author Robert HG (254963746@qq.com) on 8/15/14. * 抽象节点 @@ -28,6 +35,7 @@ public abstract class AbstractJobNode i protected T node; protected JobNodeConfig config; protected App application; + private List nodeChangeListeners; public AbstractJobNode() { application = getApplication(); @@ -35,18 +43,18 @@ public AbstractJobNode() { application.setConfig(config); // 事件中心 application.setEventCenter(new JvmEventCenter()); - this.registry = new ZkNodeRegistry(application); - // 用于master选举的监听器 - addNodeChangeListener(new MasterNodeElectionListener(application)); - // 监听自己节点变化（如，当前节点被禁用了） - addNodeChangeListener(new SelfChangeListener(application)); + application.setCommandBodyWrapper(new CommandBodyWrapper(application)); + application.setMasterElector(new MasterElector(application)); + nodeChangeListeners = new ArrayList(); } final public void start() { try { - node = NodeFactory.create(application.getPathParser(), getNodeClass(), config); + node = NodeFactory.create(getNodeClass(), config); config.setNodeType(node.getNodeType()); + initRegistry(); + LOGGER.info("当前节点配置:{}", config); // 监听节点 启用/禁用消息 @@ -86,6 +94,59 @@ final public void stop() { } } + @Override + public void destroy() { + try { + registry.destroy(); + } catch (Throwable e) { + LOGGER.error("销毁失败!", e); + } + } + + private void initRegistry() { + registry = RegistryFactory.getRegistry(application); + ((AbstractRegistry) registry).setNode(node); + // 订阅的node管理 + SubscribedNodeManager subscribedNodeManager = new SubscribedNodeManager(application); + application.setSubscribedNodeManager(subscribedNodeManager); + nodeChangeListeners.add(subscribedNodeManager); + // 用于master选举的监听器 + nodeChangeListeners.add(new MasterElectionListener(application)); + // 监听自己节点变化（如，当前节点被禁用了） + nodeChangeListeners.add(new SelfChangeListener(application)); + + registry.subscribe(node, new NotifyListener() { + private final Logger NOTIFY_LOGGER = LoggerFactory.getLogger(NotifyListener.class); + + @Override + public void notify(NotifyEvent event, List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } + switch (event) { + case ADD: + for (NodeChangeListener listener : nodeChangeListeners) { + try { + listener.addNodes(nodes); + } catch (Throwable t) { + NOTIFY_LOGGER.error("{} add nodes failed , cause: {}", listener.getClass(), t.getMessage(), t); + } + } + break; + case REMOVE: + for (NodeChangeListener listener : nodeChangeListeners) { + try { + listener.removeNodes(nodes); + } catch (Throwable t) { + NOTIFY_LOGGER.error("{} remove nodes failed , cause: {}", listener.getClass(), t.getMessage(), t); + } + } + break; + } + } + }); + } + protected abstract void innerStart(); protected abstract void innerStop(); @@ -117,10 +178,10 @@ private Class getNodeClass() { /** * 设置zookeeper注册中心地址 * - * @param zookeeperAddress + * @param registryAddress */ - public void setZookeeperAddress(String zookeeperAddress) { - config.setZookeeperAddress(zookeeperAddress); + public void setRegistryAddress(String registryAddress) { + config.setRegistryAddress(registryAddress); } /** @@ -144,19 +205,30 @@ public void setClusterName(String clusterName) { /** * 添加节点监听器 * - * @param nodeChangeListener + * @param notifyListener */ - public void addNodeChangeListener(NodeChangeListener nodeChangeListener) { - registry.addNodeChangeListener(nodeChangeListener); + public void addNodeChangeListener(NodeChangeListener notifyListener) { + if (notifyListener != null) { + nodeChangeListeners.add(notifyListener); + } } /** * 添加 master 节点变化监听器 * - * @param masterNodeChangeListener + * @param masterChangeListener */ - public void addMasterNodeChangeListener(MasterNodeChangeListener masterNodeChangeListener) { - application.getMasterElector().addMasterNodeChangeListener(masterNodeChangeListener); + public void addMasterChangeListener(MasterChangeListener masterChangeListener) { + application.getMasterElector().addMasterChangeListener(masterChangeListener); } + /** + * 设置额外的配置参数 + * + * @param key + * @param value + */ + public void addConfig(String key, String value) { + config.setParameter(key, value); + } } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/JobNode.java b/job-core/src/main/java/com/lts/job/core/cluster/JobNode.java index e39a3e843..3bec273b8 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/JobNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/JobNode.java @@ -16,4 +16,8 @@ public interface JobNode { */ public void stop(); + /** + * destroy + */ + public void destroy(); } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java b/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java index 79ad12228..36462059a 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/MasterElector.java @@ -1,7 +1,7 @@ package com.lts.job.core.cluster; import com.lts.job.core.Application; -import com.lts.job.core.listener.MasterNodeChangeListener; +import com.lts.job.core.listener.MasterChangeListener; import com.lts.job.core.util.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,22 +21,18 @@ public class MasterElector { private static final Logger LOGGER = LoggerFactory.getLogger(MasterElector.class); private Application application; - private List masterNodeChangeListenerList; + private List masterChangeListenerList; private volatile Node master; public MasterElector(Application application) { this.application = application; } - public void addMasterNodeChangeListener(MasterNodeChangeListener masterNodeChangeListener) { - if (masterNodeChangeListenerList == null) { - masterNodeChangeListenerList = new ArrayList(); + public void addMasterChangeListener(MasterChangeListener masterChangeListener) { + if (masterChangeListenerList == null) { + masterChangeListenerList = new ArrayList(); } - masterNodeChangeListenerList.add(masterNodeChangeListener); - } - - public Node getMaster() { - return master; + masterChangeListenerList.add(masterChangeListener); } public void addNodes(List nodes) { @@ -65,12 +61,18 @@ public void addNode(Node newNode) { } } - public void removeNode(Node removedNode) { - + public void removeNode(List removedNodes) { if (master != null) { - if (master.getIdentity().equals(removedNode.getIdentity())) { + boolean masterRemoved = false; + for (Node removedNode : removedNodes) { + if (master.getIdentity().equals(removedNode.getIdentity())) { + masterRemoved = true; + } + } + if (masterRemoved) { // 如果挂掉的是master, 需要重新选举 - List nodes = application.getSubscribedNodeManager().getNodeList(application.getConfig().getNodeType(), application.getConfig().getNodeGroup()); + List nodes = application.getSubscribedNodeManager(). + getNodeList(application.getConfig().getNodeType(), application.getConfig().getNodeGroup()); if (CollectionUtils.isNotEmpty(nodes)) { Node newMaster = null; for (Node node : nodes) { @@ -92,19 +94,19 @@ public void removeNode(Node removedNode) { private void notifyListener() { boolean isMaster = false; if (application.getConfig().getIdentity().equals(master.getIdentity())) { - LOGGER.info("Master节点变化为当前节点:{}", master.getPath()); + LOGGER.info("Master节点变化为当前节点:{}", master); isMaster = true; } else { - LOGGER.info("Master节点为:{}", master.getPath()); + LOGGER.info("Master节点为:{}", master); isMaster = false; } - if (masterNodeChangeListenerList != null) { - for (MasterNodeChangeListener masterNodeChangeListener : masterNodeChangeListenerList) { + if (masterChangeListenerList != null) { + for (MasterChangeListener masterChangeListener : masterChangeListenerList) { try { - masterNodeChangeListener.change(master, isMaster); + masterChangeListener.change(master, isMaster); } catch (Throwable t) { - LOGGER.error("masterNodeChangeListener通知失败!", t); + LOGGER.error("masterChangeListener通知失败!", t); } } } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/Node.java b/job-core/src/main/java/com/lts/job/core/cluster/Node.java index 06a8992ce..21b29252c 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/Node.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/Node.java @@ -5,12 +5,10 @@ /** * @author Robert HG (254963746@qq.com) on 6/22/14. - * 节点 + * 节点 */ public class Node { - // 节点路径(zookeeper 上的路径) - private String path; // 是否可用 private boolean isAvailable = true; @@ -27,14 +25,6 @@ public class Node { // 自己关注的节点类型 private List listenNodeTypes; - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - public boolean isAvailable() { return isAvailable; } @@ -117,18 +107,17 @@ public void addListenNodeType(NodeType nodeType) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof Node)) return false; + if (o == null || getClass() != o.getClass()) return false; Node node = (Node) o; - if (!identity.equals(node.identity)) return false; + return !(identity != null ? !identity.equals(node.identity) : node.identity != null); - return true; } @Override public int hashCode() { - return path != null ? path.hashCode() : 0; + return identity != null ? identity.hashCode() : 0; } public String getAddress() { @@ -138,7 +127,6 @@ public String getAddress() { @Override public String toString() { return "Node{" + - "path='" + path + '\'' + ", isAvailable=" + isAvailable + ", nodeType=" + nodeType + ", ip='" + ip + '\'' + diff --git a/job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java b/job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java index d6906f5d1..79c060e76 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/SubscribedNodeManager.java @@ -2,6 +2,7 @@ import com.lts.job.core.Application; +import com.lts.job.core.listener.NodeChangeListener; import com.lts.job.core.util.CollectionUtils; import com.lts.job.core.util.ListUtils; import org.slf4j.Logger; @@ -15,7 +16,7 @@ * @author Robert HG (254963746@qq.com) on 6/22/14. * 节点管理 (主要用于管理自己关注的节点) */ -public class SubscribedNodeManager { +public class SubscribedNodeManager implements NodeChangeListener { private static final Logger LOGGER = LoggerFactory.getLogger(SubscribedNodeManager.class); private final ConcurrentHashMap> NODES = new ConcurrentHashMap>(); @@ -26,24 +27,34 @@ public SubscribedNodeManager(Application application) { this.application = application; } - public void addNode(Node node) { - - // JobClient 和 TaskTracker 只对自己同一个组的节点关注和JobTracker节点关注 - // JobTracker 要对所有节点都要关注 - if ((NodeType.JOB_TRACKER.equals(node.getNodeType())) || - ((application.getConfig().getNodeType().equals(node.getNodeType()) - && application.getConfig().getNodeGroup().equals(node.getGroup())) - || (NodeType.JOB_TRACKER.equals(application.getConfig().getNodeType()))) - ) { + /** + * 添加监听的节点 + * + * @param node + */ + private void addNode(Node node) { + + if ((NodeType.JOB_TRACKER.equals(node.getNodeType()))) { + // 如果增加的JobTracker节点，那么直接添加，因为三种节点都需要监听 + _addNode(node); + } else if (NodeType.JOB_TRACKER.equals(application.getConfig().getNodeType())) { + // 如果当天节点是JobTracker节点，那么直接添加，因为JobTracker节点要监听三种节点 + _addNode(node); + } else if (application.getConfig().getNodeType().equals(node.getNodeType()) + && application.getConfig().getNodeGroup().equals(node.getGroup())) { + // 剩下这种情况是JobClient和TaskTracker都只监听和自己同一个group的节点 + _addNode(node); + } + } - List nodeList = NODES.get(node.getNodeType()); - if (CollectionUtils.isEmpty(nodeList)) { - nodeList = new CopyOnWriteArrayList(); - NODES.put(node.getNodeType(), nodeList); - } - nodeList.add(node); - LOGGER.info("添加节点{}", node); + private void _addNode(Node node) { + List nodeList = NODES.get(node.getNodeType()); + if (CollectionUtils.isEmpty(nodeList)) { + nodeList = new CopyOnWriteArrayList(); + NODES.put(node.getNodeType(), nodeList); } + nodeList.add(node); + LOGGER.info("添加节点{}", node); } public List getNodeList(final NodeType nodeType, final String nodeGroup) { @@ -62,29 +73,35 @@ public List getNodeList(NodeType nodeType) { return NODES.get(nodeType); } - public void removeNode(Node node) { - List nodeList = NODES.get(node.getNodeType()); - if (nodeList.remove(node)) { - LOGGER.info("删除节点{}", node); + private void removeNode(Node delNode) { + List nodeList = NODES.get(delNode.getNodeType()); + if (CollectionUtils.isNotEmpty(nodeList)) { + for (Node node : nodeList) { + if (node.getIdentity().equals(delNode.getIdentity())) { + nodeList.remove(node); + LOGGER.info("删除节点{}", node); + } + } } } - /** - * 是否包含某个元素 - * @param node - * @return - */ - public boolean contains(Node node) { - List nodes = NODES.get(node.getNodeType()); - for (Node inNode : nodes) { - if (inNode.getIdentity().equals(node.getIdentity())) { - return true; - } + @Override + public void addNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } + for (Node node : nodes) { + addNode(node); } - return false; } - public void destroy() { - NODES.clear(); + @Override + public void removeNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } + for (Node node : nodes) { + removeNode(node); + } } } diff --git a/job-core/src/main/java/com/lts/job/core/constant/Constants.java b/job-core/src/main/java/com/lts/job/core/constant/Constants.java index 2218ed44c..67fa6d104 100644 --- a/job-core/src/main/java/com/lts/job/core/constant/Constants.java +++ b/job-core/src/main/java/com/lts/job/core/constant/Constants.java @@ -24,4 +24,28 @@ public interface Constants { // 默认TaskTracker节点组 public static final String DEFAULT_NODE_TASK_TRACKER_GROUP = "taskTrackerGroup"; + public static final String CHARSET = "utf-8"; + + public static final int DEFAULT_TIMEOUT = 1000; + + public static final String TIMEOUT_KEY = "timeout"; + + public static final String SESSION_TIMEOUT_KEY = "session"; + + public static final int DEFAULT_SESSION_TIMEOUT = 60 * 1000; + + public static final String REGISTER = "register"; + + public static final String UNREGISTER = "unregister"; + + /** + * 注册中心失败事件重试事件 + */ + public static final String REGISTRY_RETRY_PERIOD_KEY = "retry.period"; + + /** + * 重试周期 + */ + public static final int DEFAULT_REGISTRY_RETRY_PERIOD = 5 * 1000; + } diff --git a/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java b/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java index b001ccdc2..a950b6443 100644 --- a/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java +++ b/job-core/src/main/java/com/lts/job/core/constant/EcTopic.java @@ -1,7 +1,7 @@ package com.lts.job.core.constant; /** - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public interface EcTopic { diff --git a/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java b/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java index 1ccecc3b8..bb3b4700b 100644 --- a/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java +++ b/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java @@ -3,6 +3,9 @@ import com.lts.job.core.cluster.NodeType; import com.lts.job.core.util.JSONUtils; +import java.util.HashMap; +import java.util.Map; + /** * @author Robert HG (254963746@qq.com) on 8/20/14. * 任务节点配置 @@ -19,8 +22,8 @@ public class JobNodeConfig { private int workThreads; // 节点类型 private NodeType nodeType; - // zookeeper 地址 - private String zookeeperAddress; + // 注册中心 地址 + private String registryAddress; // 远程连接超时时间 private int invokeTimeoutMillis; // 监听端口 @@ -30,6 +33,8 @@ public class JobNodeConfig { // 集群名字 private String clusterName; + private final Map parameters = new HashMap(); + public String getClusterName() { return clusterName; } @@ -70,12 +75,12 @@ public void setNodeType(NodeType nodeType) { this.nodeType = nodeType; } - public String getZookeeperAddress() { - return zookeeperAddress; + public String getRegistryAddress() { + return registryAddress; } - public void setZookeeperAddress(String zookeeperAddress) { - this.zookeeperAddress = zookeeperAddress; + public void setRegistryAddress(String registryAddress) { + this.registryAddress = registryAddress; } public int getInvokeTimeoutMillis() { @@ -114,6 +119,22 @@ public void setAvailable(boolean available) { this.available = available; } + public void setParameter(String key, String value) { + parameters.put(key, value); + } + + public T getParameter(String key) { + return (T)parameters.get(key); + } + + public T getParameter(String key, T defaultValue) { + Object value = parameters.get(key); + if (value == null) { + return defaultValue; + } + return (T)value; + } + @Override public String toString() { return JSONUtils.toJSONString(this); diff --git a/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java b/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java index d750ca267..fc79f1c6e 100644 --- a/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java +++ b/job-core/src/main/java/com/lts/job/core/exception/JobSubmitException.java @@ -1,7 +1,7 @@ package com.lts.job.core.exception; /** - * Created by hugui on 5/12/15. + * @author Robert HG (254963746@qq.com) on 5/12/15. */ public class JobSubmitException extends RuntimeException { diff --git a/job-core/src/main/java/com/lts/job/core/exception/NodeRegistryException.java b/job-core/src/main/java/com/lts/job/core/exception/NodeRegistryException.java new file mode 100644 index 000000000..3748c91f1 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/exception/NodeRegistryException.java @@ -0,0 +1,27 @@ +package com.lts.job.core.exception; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public class NodeRegistryException extends RuntimeException { + + public NodeRegistryException() { + super(); + } + + public NodeRegistryException(String message) { + super(message); + } + + public NodeRegistryException(String message, Throwable cause) { + super(message, cause); + } + + public NodeRegistryException(Throwable cause) { + super(cause); + } + + protected NodeRegistryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java b/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java index b28c4d339..0f7ecb63d 100644 --- a/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java +++ b/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java @@ -14,7 +14,7 @@ public static JobNodeConfig getDefaultConfig() { config.setIdentity(StringUtils.generateUUID()); config.setWorkThreads(Constants.AVAILABLE_PROCESSOR); config.setNodeGroup("lts"); - config.setZookeeperAddress("localhost:2181"); + config.setRegistryAddress("zookeeper://localhost:2181"); config.setInvokeTimeoutMillis(1000 * 6); config.setListenPort(0); config.setJobInfoSavePath(Constants.USER_HOME); diff --git a/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java b/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java index 2d99c5fd3..e3fd87ee0 100644 --- a/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java +++ b/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java @@ -2,7 +2,6 @@ import com.lts.job.core.cluster.Node; import com.lts.job.core.domain.JobNodeConfig; -import com.lts.job.core.registry.PathParser; import com.lts.job.core.util.NetUtils; /** @@ -11,7 +10,7 @@ */ public class NodeFactory { - public static T create(PathParser pathParser, Class clazz, JobNodeConfig config) { + public static T create(Class clazz, JobNodeConfig config) { try { T node = clazz.newInstance(); node.setIp(NetUtils.getLocalHost()); @@ -19,7 +18,6 @@ public static T create(PathParser pathParser, Class clazz, J node.setThreads(config.getWorkThreads()); node.setPort(config.getListenPort()); node.setIdentity(config.getIdentity()); - node.setPath(pathParser.getPath(node)); return node; } catch (InstantiationException e) { throw new RuntimeException(e); diff --git a/job-core/src/main/java/com/lts/job/core/listener/MasterNodeChangeListener.java b/job-core/src/main/java/com/lts/job/core/listener/MasterChangeListener.java similarity index 89% rename from job-core/src/main/java/com/lts/job/core/listener/MasterNodeChangeListener.java rename to job-core/src/main/java/com/lts/job/core/listener/MasterChangeListener.java index c7b9e8cdf..d5f84cdef 100644 --- a/job-core/src/main/java/com/lts/job/core/listener/MasterNodeChangeListener.java +++ b/job-core/src/main/java/com/lts/job/core/listener/MasterChangeListener.java @@ -6,7 +6,7 @@ * @author Robert HG (254963746@qq.com) on 8/23/14. * Master 节点变化 监听器 */ -public interface MasterNodeChangeListener { +public interface MasterChangeListener { /** * master 为 master节点 diff --git a/job-core/src/main/java/com/lts/job/core/listener/MasterNodeElectionListener.java b/job-core/src/main/java/com/lts/job/core/listener/MasterElectionListener.java similarity index 52% rename from job-core/src/main/java/com/lts/job/core/listener/MasterNodeElectionListener.java rename to job-core/src/main/java/com/lts/job/core/listener/MasterElectionListener.java index 72df356fa..d1f2de3ff 100644 --- a/job-core/src/main/java/com/lts/job/core/listener/MasterNodeElectionListener.java +++ b/job-core/src/main/java/com/lts/job/core/listener/MasterElectionListener.java @@ -2,38 +2,43 @@ import com.lts.job.core.Application; import com.lts.job.core.cluster.Node; +import com.lts.job.core.util.CollectionUtils; import java.util.ArrayList; import java.util.List; /** * @author Robert HG (254963746@qq.com) on 8/23/14. - * 用来监听 自己类型 节点的变化,用来选举master + * 用来监听 自己类型 节点的变化,用来选举master */ -public class MasterNodeElectionListener implements NodeChangeListener { +public class MasterElectionListener implements NodeChangeListener { private Application application; - public MasterNodeElectionListener(Application application) { + public MasterElectionListener(Application application) { this.application = application; } - @Override - public void addNode(Node node) { - if (isSameGroup(node)) { - application.getMasterElector().addNode(node); + public void removeNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; } - } - - @Override - public void removeNode(Node node) { - if (isSameGroup(node)) { - application.getMasterElector().removeNode(node); + // 只需要和当前节点相同的节点类型和组 + List groupNodes = new ArrayList(); + for (Node node : nodes) { + if (isSameGroup(node)) { + groupNodes.add(node); + } + } + if (groupNodes.size() > 0) { + application.getMasterElector().removeNode(groupNodes); } } - @Override public void addNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } // 只需要和当前节点相同的节点类型和组 List groupNodes = new ArrayList(); for (Node node : nodes) { @@ -41,18 +46,20 @@ public void addNodes(List nodes) { groupNodes.add(node); } } - if(groupNodes.size() > 0){ + if (groupNodes.size() > 0) { application.getMasterElector().addNodes(groupNodes); } } /** * 是否和当前节点是相同的GROUP + * * @param node * @return */ - private boolean isSameGroup(Node node){ + private boolean isSameGroup(Node node) { return node.getNodeType().equals(application.getConfig().getNodeType()) && node.getGroup().equals(application.getConfig().getNodeGroup()); } + } diff --git a/job-core/src/main/java/com/lts/job/core/listener/NodeChangeListener.java b/job-core/src/main/java/com/lts/job/core/listener/NodeChangeListener.java index 0d73a0ef7..08f958c66 100644 --- a/job-core/src/main/java/com/lts/job/core/listener/NodeChangeListener.java +++ b/job-core/src/main/java/com/lts/job/core/listener/NodeChangeListener.java @@ -5,28 +5,21 @@ import java.util.List; /** - * @author Robert HG (254963746@qq.com) on 8/17/14. - * 节点变化监听器 + * Created by hugui on 5/18/15. */ public interface NodeChangeListener { /** * 添加节点 * - * @param node - */ - public void addNode(Node node); - - /** - * 删除节点 - * - * @param node + * @param nodes */ - public void removeNode(Node node); + public void addNodes(List nodes); /** - * 批量添加节点 + * 移除节点 * @param nodes */ - public void addNodes(List nodes); + public void removeNodes(List nodes); + } diff --git a/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java b/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java index 578c7be37..ea33e97da 100644 --- a/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java +++ b/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java @@ -5,6 +5,7 @@ import com.lts.job.core.cluster.NodeType; import com.lts.job.core.constant.EcTopic; import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.core.util.CollectionUtils; import com.lts.job.ec.EventCenter; import com.lts.job.ec.EventInfo; @@ -12,7 +13,8 @@ /** * 用来监听自己的节点信息变化 - * Created by hugui on 5/11/15. + * + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public class SelfChangeListener implements NodeChangeListener { @@ -24,8 +26,8 @@ public SelfChangeListener(Application application) { this.eventCenter = application.getEventCenter(); } - @Override - public void addNode(Node node) { + + private void change(Node node) { if (node.getIdentity().equals(config.getIdentity())) { // 是当前节点, 看看节点配置是否发生变化 // 1. 看 threads 有没有改变 , 目前只有 TASK_TRACKER 对 threads起作用 @@ -45,12 +47,17 @@ public void addNode(Node node) { } @Override - public void removeNode(Node node) { - // do nothing + public void addNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } + for (Node node : nodes) { + change(node); + } } @Override - public void addNodes(List nodes) { - // do nothing + public void removeNodes(List nodes) { + } } diff --git a/job-core/src/main/java/com/lts/job/core/registry/AbstractRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/AbstractRegistry.java new file mode 100644 index 000000000..895275059 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/AbstractRegistry.java @@ -0,0 +1,189 @@ +package com.lts.job.core.registry; + +import com.lts.job.core.Application; +import com.lts.job.core.cluster.Node; +import com.lts.job.core.util.CollectionUtils; +import com.lts.job.core.util.ConcurrentHashSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public abstract class AbstractRegistry implements Registry { + + protected final static Logger LOGGER = LoggerFactory.getLogger(Registry.class); + + private final Set registered = new ConcurrentHashSet(); + private final ConcurrentMap> subscribed = new ConcurrentHashMap>(); + + protected Application application; + private Node node; + + public AbstractRegistry(Application application) { + + this.application = application; + } + + @Override + public void register(Node node) { + if (node == null) { + throw new IllegalArgumentException("register node == null"); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Register: " + node); + } + registered.add(node); + } + + @Override + public void unregister(Node node) { + if (node == null) { + throw new IllegalArgumentException("unregister node == null"); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Unregister: " + node); + } + registered.remove(node); + } + + @Override + public void subscribe(Node node, NotifyListener listener) { + if (node == null) { + throw new IllegalArgumentException("subscribe node == null"); + } + if (listener == null) { + throw new IllegalArgumentException("subscribe listener == null"); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Subscribe: " + node); + } + Set listeners = subscribed.get(node); + if (listeners == null) { + subscribed.putIfAbsent(node, new ConcurrentHashSet()); + listeners = subscribed.get(node); + } + listeners.add(listener); + + } + + @Override + public void unsubscribe(Node node, NotifyListener listener) { + if (node == null) { + throw new IllegalArgumentException("unsubscribe node == null"); + } + if (listener == null) { + throw new IllegalArgumentException("unsubscribe listener == null"); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("unsubscribe: " + node); + } + Set listeners = subscribed.get(node); + if (listeners != null) { + listeners.remove(listener); + } + } + + protected void notify(NotifyEvent event, List nodes, NotifyListener listener) { + if (event == null) { + throw new IllegalArgumentException("notify event == null"); + } + if (listener == null) { + throw new IllegalArgumentException("notify listener == null"); + } + if (CollectionUtils.isEmpty(nodes)) { + LOGGER.warn("Ignore empty notify nodes for subscribe node " + getNode()); + return; + } + + listener.notify(event, nodes); + } + + @Override + public void destroy() { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Destroy registry:" + getNode()); + } + Set destroyRegistered = new HashSet(getRegistered()); + if (!destroyRegistered.isEmpty()) { + for (Node node : new HashSet(getRegistered())) { + try { + unregister(node); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Destroy unregister node " + node); + } + } catch (Throwable t) { + LOGGER.warn("Failed to unregister node " + node + " to registry " + getNode() + " on destroy, cause: " + t.getMessage(), t); + } + } + } + Map> destroySubscribed = new HashMap>(getSubscribed()); + if (!destroySubscribed.isEmpty()) { + for (Map.Entry> entry : destroySubscribed.entrySet()) { + Node node = entry.getKey(); + for (NotifyListener listener : entry.getValue()) { + try { + unsubscribe(node, listener); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Destroy unsubscribe node " + node); + } + } catch (Throwable t) { + LOGGER.warn("Failed to unsubscribe node " + node + " to registry " + getNode() + " on destroy, cause: " + t.getMessage(), t); + } + } + } + } + } + + protected Set getRegistered() { + return registered; + } + + protected ConcurrentMap> getSubscribed() { + return subscribed; + } + + public Node getNode() { + return node; + } + + public void setNode(Node node) { + this.node = node; + } + + /** + * 恢复 + * + * @throws Exception + */ + protected void recover() throws Exception { + // register + Set recoverRegistered = new HashSet(getRegistered()); + if (!recoverRegistered.isEmpty()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Recover register node " + recoverRegistered); + } + for (Node node : recoverRegistered) { + register(node); + } + } + // subscribe + Map> recoverSubscribed = new HashMap>(getSubscribed()); + if (!recoverSubscribed.isEmpty()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Recover subscribe node " + recoverSubscribed.keySet()); + } + for (Map.Entry> entry : recoverSubscribed.entrySet()) { + Node node = entry.getKey(); + for (NotifyListener listener : entry.getValue()) { + subscribe(node, listener); + } + } + } + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/FailbackRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/FailbackRegistry.java new file mode 100644 index 000000000..d02504ca7 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/FailbackRegistry.java @@ -0,0 +1,325 @@ +package com.lts.job.core.registry; + +import com.lts.job.core.Application; +import com.lts.job.core.cluster.Node; +import com.lts.job.core.constant.Constants; +import com.lts.job.core.factory.NamedThreadFactory; +import com.lts.job.core.util.ConcurrentHashSet; + +import java.util.*; +import java.util.concurrent.*; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public abstract class FailbackRegistry extends AbstractRegistry { + + // 定时任务执行器 + private final ScheduledExecutorService retryExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("LTSRegistryFailedRetryTimer", true)); + + // 失败重试定时器，定时检查是否有请求失败，如有，无限次重试 + private ScheduledFuture retryFuture; + + // 注册失败的定时重试 + private final Set failedRegistered = new ConcurrentHashSet(); + private final Set failedUnRegistered = new ConcurrentHashSet(); + private final ConcurrentMap> failedSubscribed = new ConcurrentHashMap>(); + private final ConcurrentMap> failedUnsubscribed = new ConcurrentHashMap>(); + private final ConcurrentMap>>> failedNotified = new ConcurrentHashMap>>>(); + + public FailbackRegistry(Application application) { + super(application); + + int retryPeriod = application.getConfig().getParameter(Constants.REGISTRY_RETRY_PERIOD_KEY, Constants.DEFAULT_REGISTRY_RETRY_PERIOD); + + this.retryFuture = retryExecutor.scheduleWithFixedDelay(new Runnable() { + public void run() { + // 检测并连接注册中心 + try { + retry(); + } catch (Throwable t) { // 防御性容错 + LOGGER.error("Unexpected error occur at failed retry, cause: " + t.getMessage(), t); + } + } + }, retryPeriod, retryPeriod, TimeUnit.MILLISECONDS); + } + + @Override + public void register(Node node) { + try { + super.register(node); + failedRegistered.clear(); + doRegister(node); + } catch (Exception e) { + // 将失败的注册请求记录到失败列表，定时重试 + failedRegistered.add(node); + } + } + + @Override + public void unregister(Node node) { + try { + super.unregister(node); + failedUnRegistered.clear(); + doUnRegister(node); + } catch (Exception e) { + // 将失败的取消注册请求记录到失败列表，定时重试 + failedUnRegistered.add(node); + } + } + + @Override + public void subscribe(Node node, NotifyListener listener) { + try { + super.subscribe(node, listener); + + removeFailedSubscribed(node, listener); + + doSubscribe(node, listener); + + } catch (Exception e) { + addFailedSubscribed(node, listener); + } + } + + @Override + public void unsubscribe(Node node, NotifyListener listener) { + try { + super.unsubscribe(node, listener); + + removeFailedSubscribed(node, listener); + + doUnsubscribe(node, listener); + + } catch (Exception e) { + addFailedUnsubscribed(node, listener); + } + } + + protected void addFailedUnsubscribed(Node node, NotifyListener listener) { + // 将失败的取消订阅请求记录到失败列表，定时重试 + Set listeners = failedUnsubscribed.get(node); + if (listeners == null) { + failedUnsubscribed.putIfAbsent(node, new ConcurrentHashSet()); + listeners = failedUnsubscribed.get(node); + } + listeners.add(listener); + } + + @Override + protected void notify(NotifyEvent event, List nodes, NotifyListener listener) { + try { + super.notify(event, nodes, listener); + } catch (Exception e) { + // 将失败的通知请求记录到失败列表，定时重试 + Map>> listeners = failedNotified.get(getNode()); + + if (listeners == null) { + failedNotified.putIfAbsent(getNode(), new ConcurrentHashMap>>()); + listeners = failedNotified.get(getNode()); + } + listeners.put(listener, new NotifyPair>(event, nodes)); + LOGGER.error("Failed to notify, waiting for retry, cause: " + e.getMessage(), e); + } + } + + @Override + public void destroy() { + super.destroy(); + try { + retryFuture.cancel(true); + } catch (Throwable t) { + LOGGER.warn(t.getMessage(), t); + } + } + + @Override + protected void recover() throws Exception { + // register + Set recoverRegistered = new HashSet(getRegistered()); + if (!recoverRegistered.isEmpty()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Recover register node " + recoverRegistered); + } + for (Node node : recoverRegistered) { + failedRegistered.add(node); + } + } + // subscribe + Map> recoverSubscribed = new HashMap>(getSubscribed()); + if (!recoverSubscribed.isEmpty()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Recover subscribe node " + recoverSubscribed.keySet()); + } + for (Map.Entry> entry : recoverSubscribed.entrySet()) { + Node node = entry.getKey(); + for (NotifyListener listener : entry.getValue()) { + addFailedSubscribed(node, listener); + } + } + } + } + + private void removeFailedSubscribed(Node node, NotifyListener listener) { + Set listeners = failedSubscribed.get(node); + if (listeners != null) { + listeners.remove(listener); + } + listeners = failedUnsubscribed.get(node); + if (listeners != null) { + listeners.remove(listener); + } + Map>> notified = failedNotified.get(node); + if (notified != null) { + notified.remove(listener); + } + } + + private void addFailedSubscribed(Node node, NotifyListener listener) { + Set listeners = failedSubscribed.get(node); + if (listeners == null) { + failedSubscribed.putIfAbsent(node, new ConcurrentHashSet()); + listeners = failedSubscribed.get(node); + } + listeners.add(listener); + } + + protected void retry() { + if (!failedRegistered.isEmpty()) { + Set failed = new HashSet(); + if (failed.size() > 0) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Retry register {}", failed); + } + try { + for (Node node : failed) { + doRegister(node); + } + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry register " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + if (!failedUnRegistered.isEmpty()) { + Set failed = new HashSet(); + if (failed.size() > 0) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Retry unregister {}", failed); + } + try { + for (Node node : failed) { + doUnRegister(node); + } + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry unregister " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + if (!failedSubscribed.isEmpty()) { + Map> failed = new HashMap>(failedSubscribed); + for (Map.Entry> entry : new HashMap>(failed).entrySet()) { + if (entry.getValue() == null || entry.getValue().size() == 0) { + failed.remove(entry.getKey()); + } + } + if (failed.size() > 0) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Retry subscribe " + failed); + } + try { + for (Map.Entry> entry : failed.entrySet()) { + Node node = entry.getKey(); + Set listeners = entry.getValue(); + for (NotifyListener listener : listeners) { + try { + doSubscribe(node, listener); + listeners.remove(listener); + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry subscribe " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry subscribe " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + if (!failedUnsubscribed.isEmpty()) { + Map> failed = new HashMap>(failedUnsubscribed); + for (Map.Entry> entry : new HashMap>(failed).entrySet()) { + if (entry.getValue() == null || entry.getValue().size() == 0) { + failed.remove(entry.getKey()); + } + } + if (failed.size() > 0) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Retry unsubscribe " + failed); + } + try { + for (Map.Entry> entry : failed.entrySet()) { + Node node = entry.getKey(); + Set listeners = entry.getValue(); + for (NotifyListener listener : listeners) { + try { + doUnsubscribe(node, listener); + listeners.remove(listener); + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry unsubscribe " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry unsubscribe " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + if (!failedNotified.isEmpty()) { + Map>>> failed = new HashMap>>>(failedNotified); + for (Map.Entry>>> entry : new HashMap>>>(failed).entrySet()) { + if (entry.getValue() == null || entry.getValue().size() == 0) { + failed.remove(entry.getKey()); + } + } + if (failed.size() > 0) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Retry notify " + failed); + } + try { + for (Map>> values : failed.values()) { + for (Map.Entry>> entry : values.entrySet()) { + try { + NotifyListener listener = entry.getKey(); + NotifyPair> notifyPair = entry.getValue(); + listener.notify(notifyPair.event, notifyPair.nodes); + values.remove(listener); + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry notify " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + } catch (Throwable t) { // 忽略所有异常，等待下次重试 + LOGGER.warn("Failed to retry notify " + failed + ", waiting for again, cause: " + t.getMessage(), t); + } + } + } + } + + private class NotifyPair { + T1 event; + T2 nodes; + + public NotifyPair(T1 event, T2 nodes) { + this.event = event; + this.nodes = nodes; + } + } + + protected abstract void doRegister(Node node); + + protected abstract void doUnRegister(Node node); + + protected abstract void doSubscribe(Node node, NotifyListener listener); + + protected abstract void doUnsubscribe(Node node, NotifyListener listener); +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java b/job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java similarity index 81% rename from job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java rename to job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java index a49a11917..79f5ee4e0 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeUtils.java +++ b/job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java @@ -8,24 +8,26 @@ /** * @author Robert HG (254963746@qq.com) on 5/11/15. - *

- * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER - *

+ *

+ * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER + *

*/ -public class ZkNodeUtils { +public class NodeRegistryUtils { - public static String getBasePath(String clusterName) { + public static String getRootPath(String clusterName) { return "/LTS/" + clusterName + "/NODES"; } + public static String getNodeTypePath(String clusterName, NodeType nodeType) { + return NodeRegistryUtils.getRootPath(clusterName) + "/" + nodeType; + } + public static Node parse(String clusterName, String fullPath) { Node node = new Node(); - - node.setPath(fullPath); - String nodeType = getMatcher(getBasePath(clusterName) + "/(.*)/", fullPath); + String nodeType = getMatcher(getRootPath(clusterName) + "/(.*)/", fullPath); node.setNodeType(NodeType.valueOf(nodeType)); - String url = getMatcher(getBasePath(clusterName) + "/" + nodeType + "/" + nodeType + ":\\\\\\\\(.*)", fullPath); + String url = getMatcher(getRootPath(clusterName) + "/" + nodeType + "/" + nodeType + ":\\\\\\\\(.*)", fullPath); String address = url.split("\\?")[0]; String ip = address.split(":")[0]; @@ -64,9 +66,9 @@ public static Node parse(String clusterName, String fullPath) { return node; } - public static String getPath(String clusterName, Node node) { + public static String getFullPath(String clusterName, Node node) { StringBuilder path = new StringBuilder(); - path.append(getBasePath(clusterName)) + path.append(getRootPath(clusterName)) .append("/") .append(node.getNodeType()) .append("/") diff --git a/job-core/src/main/java/com/lts/job/core/registry/NotifyEvent.java b/job-core/src/main/java/com/lts/job/core/registry/NotifyEvent.java new file mode 100644 index 000000000..a13d52395 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/NotifyEvent.java @@ -0,0 +1,10 @@ +package com.lts.job.core.registry; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public enum NotifyEvent { + + ADD, + REMOVE +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/NotifyListener.java b/job-core/src/main/java/com/lts/job/core/registry/NotifyListener.java new file mode 100644 index 000000000..9d4f4981a --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/NotifyListener.java @@ -0,0 +1,14 @@ +package com.lts.job.core.registry; + +import com.lts.job.core.cluster.Node; + +import java.util.List; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public interface NotifyListener { + + void notify(NotifyEvent event, List nodes); + +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/PathParser.java b/job-core/src/main/java/com/lts/job/core/registry/PathParser.java deleted file mode 100644 index 54225be4e..000000000 --- a/job-core/src/main/java/com/lts/job/core/registry/PathParser.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.lts.job.core.registry; - -import com.lts.job.core.cluster.Node; - -/** - * @author Robert HG (254963746@qq.com) on 3/27/15. - */ -public interface PathParser { - - public Node parse(String path); - - public String getPath(Node node); - -} diff --git a/job-core/src/main/java/com/lts/job/core/registry/Registry.java b/job-core/src/main/java/com/lts/job/core/registry/Registry.java index c8affd58f..fe2772e3c 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/Registry.java +++ b/job-core/src/main/java/com/lts/job/core/registry/Registry.java @@ -1,11 +1,10 @@ package com.lts.job.core.registry; import com.lts.job.core.cluster.Node; -import com.lts.job.core.listener.NodeChangeListener; /** * @author Robert HG (254963746@qq.com) on 6/22/14. - * 节点注册接口 + * 节点注册接口 */ public interface Registry { @@ -24,11 +23,22 @@ public interface Registry { void unregister(Node node); /** - * 添加节点变化监听器 + * 监听节点 + * * @param listener */ - void addNodeChangeListener(NodeChangeListener listener); + void subscribe(Node node, NotifyListener listener); - void destroy(); + /** + * 取消监听节点 + * + * @param node + * @param listener + */ + void unsubscribe(Node node, NotifyListener listener); + /** + * 销毁 + */ + void destroy(); } diff --git a/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java b/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java new file mode 100644 index 000000000..d4b3a8cdd --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java @@ -0,0 +1,26 @@ +package com.lts.job.core.registry; + +import com.lts.job.core.Application; +import com.lts.job.core.registry.redis.RedisRegistry; +import com.lts.job.core.registry.zookeeper.ZookeeperRegistry; +import com.lts.job.core.util.StringUtils; + +/** + * Created by hugui on 5/17/15. + */ +public class RegistryFactory { + + public static Registry getRegistry(Application application) { + String address = application.getConfig().getRegistryAddress(); + if (StringUtils.isEmpty(address)) { + throw new IllegalArgumentException("address is null！"); + } + if (address.startsWith("zookeeper://")) { + return new ZookeeperRegistry(application); + } else if (address.startsWith("redis://")) { + return new RedisRegistry(application); + } + throw new IllegalArgumentException("illegal address protocol"); + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java deleted file mode 100644 index eeb17b842..000000000 --- a/job-core/src/main/java/com/lts/job/core/registry/ZkNodeRegistry.java +++ /dev/null @@ -1,173 +0,0 @@ -package com.lts.job.core.registry; - -import com.lts.job.core.Application; -import com.lts.job.core.cluster.Node; -import com.lts.job.core.cluster.NodeType; -import com.lts.job.core.listener.NodeChangeListener; -import com.lts.job.core.util.CollectionUtils; -import com.lts.job.registry.zookeeper.ChildListener; -import com.lts.job.registry.zookeeper.StateListener; -import com.lts.job.registry.zookeeper.ZookeeperClient; -import com.lts.job.registry.zookeeper.zkclient.ZkClientZookeeperClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author Robert HG (254963746@qq.com) on 6/22/14. - * 节点注册器，并监听自己关注的节点 - */ -public class ZkNodeRegistry implements Registry { - - private static final Logger LOGGER = LoggerFactory.getLogger(ZkNodeRegistry.class); - private ZookeeperClient zkClient; - // 用来记录父节点下的子节点的变化 - private final ConcurrentHashMap> NODE_CHILDREN_MAP = new ConcurrentHashMap>(); - private ChildChangeListener listener; - private List nodeChangeListeners; - private Application application; - private ZkPathParser zkPathParser; - - public ZkNodeRegistry(Application application) { - this.listener = new ChildChangeListener(); - this.application = application; - this.zkPathParser = new ZkPathParser(application); - application.setPathParser(this.zkPathParser); - } - - /** - * 添加节点变化监听器 - * @param nodeChangeListener - */ - public void addNodeChangeListener(NodeChangeListener nodeChangeListener) { - if (this.nodeChangeListeners == null) { - this.nodeChangeListeners = new ArrayList(); - } - this.nodeChangeListeners.add(nodeChangeListener); - } - - @Override - public void register(final Node node) { - zkClient = new ZkClientZookeeperClient(application.getConfig().getZookeeperAddress()); - zkClient.addStateListener(new StateListener() { - @Override - public void stateChanged(int state) { - if (state == RECONNECTED) { - try { - doSubscribe(node); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); - } - } - } - }); - doSubscribe(node); - } - - protected void doSubscribe(Node node) { - - if (zkClient.exists(node.getPath())) { - return; - } - - zkClient.create(node.getPath(), true, false); - - List listenNodeTypes = node.getListenNodeTypes(); - if (CollectionUtils.isNotEmpty(listenNodeTypes)) { - - for (NodeType nodeType : listenNodeTypes) { - String listenNodePath = zkPathParser.getPath(nodeType); - // 为自己关注的 节点 添加监听 - zkClient.addChildListener(listenNodePath, listener); - - // 将自己关注的 节点类型加入到节点管理中去 - List children = zkClient.getChildren(listenNodePath); - if (CollectionUtils.isNotEmpty(children)) { - List listenedNodes = new ArrayList(); - for (String child : children) { - Node listenedNode = zkPathParser.parse(listenNodePath + "/" + child); - listenedNodes.add(listenedNode); - application.getSubscribedNodeManager().addNode(listenedNode); - } - if (CollectionUtils.isNotEmpty(nodeChangeListeners)) { - for (NodeChangeListener nodeChangeListener : nodeChangeListeners) { - nodeChangeListener.addNodes(listenedNodes); - } - } - NODE_CHILDREN_MAP.put(listenNodePath, children); - } - } - } - } - - protected void doUnSubscribe(Node node) { - zkClient.delete(node.getPath()); - - List listenNodeTypes = node.getListenNodeTypes(); - if (CollectionUtils.isNotEmpty(listenNodeTypes)) { - for (NodeType nodeType : listenNodeTypes) { - zkClient.removeChildListener(zkPathParser.getPath(nodeType), listener); - - application.getSubscribedNodeManager().destroy(); - } - } - } - - @Override - public void unregister(Node node) { - doUnSubscribe(node); - zkClient.close(); - } - - @Override - public void destroy() { - try { - zkClient.close(); - } catch (Throwable t) { - LOGGER.error(t.getMessage(), t); - } - } - - private class ChildChangeListener implements ChildListener { - - @Override - public void childChanged(String path, List children) { - - List oldChildren = NODE_CHILDREN_MAP.get(path); - // 1. 找出增加的 节点 - List addChildren = CollectionUtils.getLeftDiff(children, oldChildren); - // 2. 找出减少的 节点 - List decChildren = CollectionUtils.getLeftDiff(oldChildren, children); - - if (CollectionUtils.isNotEmpty(addChildren)) { - for (String child : addChildren) { - Node node = zkPathParser.parse(path + "/" + child); - - application.getSubscribedNodeManager().addNode(node); - if (CollectionUtils.isNotEmpty(nodeChangeListeners)) { - for (NodeChangeListener nodeChangeListener : nodeChangeListeners) { - nodeChangeListener.addNode(node); - } - } - } - } - - if (CollectionUtils.isNotEmpty(decChildren)) { - for (String child : decChildren) { - Node node = zkPathParser.parse(path + "/" + child); - application.getSubscribedNodeManager().removeNode(node); - if (CollectionUtils.isNotEmpty(nodeChangeListeners)) { - for (NodeChangeListener nodeChangeListener : nodeChangeListeners) { - nodeChangeListener.removeNode(node); - } - } - } - } - NODE_CHILDREN_MAP.put(path, children); - } - } -} - diff --git a/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java b/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java deleted file mode 100644 index b9eb8a4af..000000000 --- a/job-core/src/main/java/com/lts/job/core/registry/ZkPathParser.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.lts.job.core.registry; - - -import com.lts.job.core.Application; -import com.lts.job.core.cluster.Node; -import com.lts.job.core.cluster.NodeType; - -/** - * @author Robert HG (254963746@qq.com) on 6/23/14. - *

- * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER - *

- * 节点 zookeeper上路径解析工具 - */ -public class ZkPathParser implements PathParser { - - private Application application; - - public ZkPathParser(Application application) { - this.application = application; - } - - public Node parse(String fullPath) { - return ZkNodeUtils.parse(application.getConfig().getClusterName(), fullPath); - } - - public String getPath(Node node) { - return ZkNodeUtils.getPath(application.getConfig().getClusterName(), node); - } - - public String getPath(NodeType nodeType) { - return ZkNodeUtils.getBasePath(application.getConfig().getClusterName()) + "/" + nodeType; - } - -} diff --git a/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java new file mode 100644 index 000000000..52190bcb5 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java @@ -0,0 +1,188 @@ +package com.lts.job.core.registry.redis; + +import com.lts.job.core.Application; +import com.lts.job.core.cluster.Node; +import com.lts.job.core.constant.Constants; +import com.lts.job.core.exception.NodeRegistryException; +import com.lts.job.core.factory.NamedThreadFactory; +import com.lts.job.core.registry.FailbackRegistry; +import com.lts.job.core.registry.NodeRegistryUtils; +import com.lts.job.core.registry.NotifyListener; +import org.apache.commons.pool.impl.GenericObjectPool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.util.Map; +import java.util.concurrent.*; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public class RedisRegistry extends FailbackRegistry { + + private static final Logger LOGGER = LoggerFactory.getLogger(RedisRegistry.class); + + private final Map jedisPools = new ConcurrentHashMap(); + + private String clusterName; + private final ScheduledExecutorService expireExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("LTSRedisRegistryExpireTimer", true)); + private final ScheduledFuture expireFuture; + private final int expirePeriod; + private boolean replicate; + + public RedisRegistry(Application application) { + super(application); + + this.clusterName = application.getConfig().getClusterName(); + + GenericObjectPool.Config config = new GenericObjectPool.Config(); + // TODO 可以设置n多参数 + + String address = application.getConfig().getRegistryAddress(); + + String cluster = application.getConfig().getParameter("cluster", "failover"); + if (!"failover".equals(cluster) && !"replicate".equals(cluster)) { + throw new IllegalArgumentException("Unsupported redis cluster: " + cluster + ". The redis cluster only supported failover or replicate."); + } + replicate = "replicate".equals(cluster); + + int i = address.indexOf(':'); + String host = address.substring(0, i); + int port = Integer.parseInt(address.substring(i + 1)); + this.jedisPools.put(address, new JedisPool(config, host, port, + Constants.DEFAULT_TIMEOUT)); + + this.expirePeriod = application.getConfig().getParameter(Constants.SESSION_TIMEOUT_KEY, Constants.DEFAULT_SESSION_TIMEOUT); + + this.expireFuture = expireExecutor.scheduleWithFixedDelay(new Runnable() { + public void run() { + try { + deferExpired(); // 延长过期时间 + } catch (Throwable t) { // 防御性容错 + LOGGER.error("Unexpected exception occur at defer expire time, cause: " + t.getMessage(), t); + } + } + }, expirePeriod / 2, expirePeriod / 2, TimeUnit.MILLISECONDS); + } + + private void deferExpired() { + for (Map.Entry entry : jedisPools.entrySet()) { +// JedisPool jedisPool = entry.getValue(); +// try { +// Jedis jedis = jedisPool.getResource(); +// try { +// for (URL url : new HashSet(getRegistered())) { +// if (url.getParameter(Constants.DYNAMIC_KEY, true)) { +// String key = toCategoryPath(url); +// if (jedis.hset(key, url.toFullString(), String.valueOf(System.currentTimeMillis() + expirePeriod)) == 1) { +// jedis.publish(key, Constants.REGISTER); +// } +// } +// } +// if (admin) { +// clean(jedis); +// } +// if (! replicate) { +// break;//  如果服务器端已同步数据，只需写入单台机器 +// } +// } finally { +// jedisPool.returnResource(jedis); +// } +// } catch (Throwable t) { +// logger.warn("Failed to write provider heartbeat to redis registry. registry: " + entry.getKey() + ", cause: " + t.getMessage(), t); +// } + } + } + + @Override + public void register(Node node) { + String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType()); + String value = NodeRegistryUtils.getFullPath(clusterName, node); + String expire = String.valueOf(System.currentTimeMillis() + expirePeriod); + boolean success = false; + NodeRegistryException exception = null; + for (Map.Entry entry : jedisPools.entrySet()) { + JedisPool jedisPool = entry.getValue(); + try { + Jedis jedis = jedisPool.getResource(); + try { + jedis.hset(key, value, expire); + jedis.publish(key, Constants.REGISTER); + success = true; + if (!replicate) { + break; //  如果服务器端已同步数据，只需写入单台机器 + } + } finally { + jedisPool.returnResource(jedis); + } + } catch (Throwable t) { + exception = new NodeRegistryException("Failed to register node to redis registry. registry: " + entry.getKey() + ", node: " + node + ", cause: " + t.getMessage(), t); + } + } + if (exception != null) { + if (success) { + LOGGER.warn(exception.getMessage(), exception); + } else { + throw exception; + } + } + + } + + @Override + public void unregister(Node node) { + String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType()); + String value = NodeRegistryUtils.getFullPath(clusterName, node); + boolean success = false; + NodeRegistryException exception = null; + for (Map.Entry entry : jedisPools.entrySet()) { + JedisPool jedisPool = entry.getValue(); + try { + Jedis jedis = jedisPool.getResource(); + try { + jedis.hdel(key, value); + jedis.publish(key, Constants.UNREGISTER); + success = true; + if (!replicate) { + break; //  如果服务器端已同步数据，只需写入单台机器 + } + } finally { + jedisPool.returnResource(jedis); + } + } catch (Throwable t) { + exception = new NodeRegistryException("Failed to unregister node to redis registry. registry: " + entry.getKey() + ", node: " + node + ", cause: " + t.getMessage(), t); + } + } + if (exception != null) { + if (success) { + LOGGER.warn(exception.getMessage(), exception); + } else { + throw exception; + } + } + } + + @Override + protected void doRegister(Node node) { + + } + + @Override + protected void doUnRegister(Node node) { + + } + + @Override + protected void doSubscribe(Node node, NotifyListener listener) { + + } + + @Override + protected void doUnsubscribe(Node node, NotifyListener listener) { + + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java new file mode 100644 index 000000000..257d04605 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java @@ -0,0 +1,174 @@ +package com.lts.job.core.registry.zookeeper; + +import com.lts.job.core.Application; +import com.lts.job.core.cluster.Node; +import com.lts.job.core.cluster.NodeType; +import com.lts.job.core.registry.FailbackRegistry; +import com.lts.job.core.registry.NodeRegistryUtils; +import com.lts.job.core.registry.NotifyEvent; +import com.lts.job.core.registry.NotifyListener; +import com.lts.job.core.util.CollectionUtils; +import com.lts.job.zookeeper.ChildListener; +import com.lts.job.zookeeper.StateListener; +import com.lts.job.zookeeper.ZookeeperClient; +import com.lts.job.zookeeper.zkclient.ZkClientZookeeperClient; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Robert HG (254963746@qq.com) on 6/22/14. + * 节点注册器，并监听自己关注的节点 + */ +public class ZookeeperRegistry extends FailbackRegistry { + + private ZookeeperClient zkClient; + // 用来记录父节点下的子节点的变化 + private final ConcurrentHashMap> childrenNodeMap; + + private final ConcurrentMap> zkListeners; + + private String clusterName; + + public ZookeeperRegistry(Application application) { + super(application); + this.clusterName = application.getConfig().getClusterName(); + this.childrenNodeMap = new ConcurrentHashMap>(); + + String address = application.getConfig().getRegistryAddress().replace("zookeeper://", ""); + this.zkClient = new ZkClientZookeeperClient(address); + this.zkListeners = new ConcurrentHashMap>(); + zkClient.addStateListener(new StateListener() { + @Override + public void stateChanged(int state) { + if (state == RECONNECTED) { + try { + recover(); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + } + }); + } + + @Override + protected void doRegister(Node node) { + String fullPath = NodeRegistryUtils.getFullPath(clusterName, node); + if (zkClient.exists(fullPath)) { + return; + } + zkClient.create(fullPath, true, false); + } + + @Override + protected void doUnRegister(Node node) { + String fullPath = NodeRegistryUtils.getFullPath(clusterName, node); + zkClient.delete(fullPath); + } + + @Override + protected void doSubscribe(Node node, NotifyListener listener) { + List listenNodeTypes = node.getListenNodeTypes(); + if (CollectionUtils.isEmpty(listenNodeTypes)) { + return; + } + for (NodeType listenNodeType : listenNodeTypes) { + String listenNodePath = NodeRegistryUtils.getNodeTypePath(clusterName, listenNodeType); + + ChildListener zkListener = addZkListener(node, listener); + + // 为自己关注的 节点 添加监听 + zkClient.addChildListener(listenNodePath, zkListener); + + // 将自己关注的 节点类型加入到节点管理中去 + List children = zkClient.getChildren(listenNodePath); + if (CollectionUtils.isNotEmpty(children)) { + List listenedNodes = new ArrayList(); + for (String child : children) { + Node listenedNode = NodeRegistryUtils.parse(clusterName, listenNodePath + "/" + child); + listenedNodes.add(listenedNode); + } + notify(NotifyEvent.ADD, listenedNodes, listener); + childrenNodeMap.put(listenNodePath, children); + } + } + } + + @Override + protected void doUnsubscribe(Node node, NotifyListener listener) { + ConcurrentMap listeners = zkListeners.get(node); + if (listeners != null) { + ChildListener zkListener = listeners.get(listener); + if (zkListener != null) { + List listenNodeTypes = node.getListenNodeTypes(); + if (CollectionUtils.isEmpty(listenNodeTypes)) { + return; + } + for (NodeType listenNodeType : listenNodeTypes) { + String listenNodePath = NodeRegistryUtils.getNodeTypePath(clusterName, listenNodeType); + zkClient.removeChildListener(listenNodePath, zkListener); + } + } + } + } + + + private ChildListener addZkListener(Node node, final NotifyListener listener) { + + ConcurrentMap listeners = zkListeners.get(node); + if (listeners == null) { + zkListeners.putIfAbsent(node, new ConcurrentHashMap()); + listeners = zkListeners.get(node); + } + ChildListener zkListener = listeners.get(listener); + if (zkListener == null) { + + listeners.putIfAbsent(listener, new ChildListener() { + + public void childChanged(String parentPath, List currentChilds) { + List oldChilds = childrenNodeMap.get(parentPath); + // 1. 找出增加的 节点 + List addChilds = CollectionUtils.getLeftDiff(currentChilds, oldChilds); + // 2. 找出减少的 节点 + List decChilds = CollectionUtils.getLeftDiff(oldChilds, currentChilds); + + if (CollectionUtils.isNotEmpty(addChilds)) { + + List nodes = new ArrayList(addChilds.size()); + for (String child : addChilds) { + Node node = NodeRegistryUtils.parse(clusterName, parentPath + "/" + child); + nodes.add(node); + } + ZookeeperRegistry.this.notify(NotifyEvent.ADD, nodes, listener); + } + + if (CollectionUtils.isNotEmpty(decChilds)) { + List nodes = new ArrayList(addChilds.size()); + for (String child : decChilds) { + Node node = NodeRegistryUtils.parse(clusterName, parentPath + "/" + child); + nodes.add(node); + } + ZookeeperRegistry.this.notify(NotifyEvent.REMOVE, nodes, listener); + } + childrenNodeMap.put(parentPath, currentChilds); + } + }); + zkListener = listeners.get(listener); + } + return zkListener; + } + + @Override + public void destroy() { + super.destroy(); + try { + zkClient.close(); + } catch (Exception e) { + LOGGER.warn("Failed to close zookeeper client " + getNode() + ", cause: " + e.getMessage(), e); + } + } +} + diff --git a/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java b/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java index 551a05925..27f88618c 100644 --- a/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java +++ b/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java @@ -38,8 +38,7 @@ public HeartBeatMonitor(RemotingClientDelegate remotingClient, Application appli public void start() { HEART_BEAT_EXECUTOR_SERVICE.scheduleWithFixedDelay( - new HeartBeat(), 5, 30, TimeUnit.SECONDS); // 30s 一次心跳 - + new HeartBeat(), 3, 30, TimeUnit.SECONDS); // 30s 一次心跳 } public void stop() { diff --git a/job-core/src/main/java/com/lts/job/ec/EventCenter.java b/job-core/src/main/java/com/lts/job/ec/EventCenter.java index 2870408f2..c605fe579 100644 --- a/job-core/src/main/java/com/lts/job/ec/EventCenter.java +++ b/job-core/src/main/java/com/lts/job/ec/EventCenter.java @@ -2,7 +2,7 @@ /** * 事件中心接口 - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public interface EventCenter { diff --git a/job-core/src/main/java/com/lts/job/ec/EventInfo.java b/job-core/src/main/java/com/lts/job/ec/EventInfo.java index baf04d038..e15e02a5a 100644 --- a/job-core/src/main/java/com/lts/job/ec/EventInfo.java +++ b/job-core/src/main/java/com/lts/job/ec/EventInfo.java @@ -5,7 +5,7 @@ /** * 事件信息 - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public class EventInfo { diff --git a/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java b/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java index 223e3151f..f7602e1e6 100644 --- a/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java +++ b/job-core/src/main/java/com/lts/job/ec/EventSubscriber.java @@ -2,7 +2,7 @@ /** * 事件订阅者 - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public class EventSubscriber { diff --git a/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java b/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java index 3d21d845c..5eb4ae15e 100644 --- a/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java +++ b/job-core/src/main/java/com/lts/job/ec/JvmEventCenter.java @@ -14,7 +14,7 @@ /** * 在一个jvm中的pub sub 简易实现 - * Created by hugui on 5/12/15. + * @author Robert HG (254963746@qq.com) on 5/12/15. */ public class JvmEventCenter implements EventCenter{ diff --git a/job-core/src/main/java/com/lts/job/ec/Observer.java b/job-core/src/main/java/com/lts/job/ec/Observer.java index 4d98c5213..dbb06ed86 100644 --- a/job-core/src/main/java/com/lts/job/ec/Observer.java +++ b/job-core/src/main/java/com/lts/job/ec/Observer.java @@ -2,7 +2,7 @@ /** * 事件观察者接口 - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public interface Observer { diff --git a/job-core/src/main/java/com/lts/job/remoting/netty/NettyClientConfig.java b/job-core/src/main/java/com/lts/job/remoting/netty/NettyClientConfig.java index 18f0877be..b2bbb8102 100644 --- a/job-core/src/main/java/com/lts/job/remoting/netty/NettyClientConfig.java +++ b/job-core/src/main/java/com/lts/job/remoting/netty/NettyClientConfig.java @@ -1,12 +1,14 @@ package com.lts.job.remoting.netty; +import com.lts.job.core.constant.Constants; + /** * Netty客户端配置类 */ public class NettyClientConfig { // 处理Server Response/Request private int clientWorkerThreads = 4; - private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors(); + private int clientCallbackExecutorThreads = Constants.AVAILABLE_PROCESSOR; private int clientSelectorThreads = 1; private int clientOnewaySemaphoreValue = 256; private int clientAsyncSemaphoreValue = 128; diff --git a/job-core/src/main/java/com/lts/job/registry/zookeeper/ChildListener.java b/job-core/src/main/java/com/lts/job/zookeeper/ChildListener.java similarity index 81% rename from job-core/src/main/java/com/lts/job/registry/zookeeper/ChildListener.java rename to job-core/src/main/java/com/lts/job/zookeeper/ChildListener.java index 634606aa5..c60a61204 100755 --- a/job-core/src/main/java/com/lts/job/registry/zookeeper/ChildListener.java +++ b/job-core/src/main/java/com/lts/job/zookeeper/ChildListener.java @@ -1,4 +1,4 @@ -package com.lts.job.registry.zookeeper; +package com.lts.job.zookeeper; import java.util.List; diff --git a/job-core/src/main/java/com/lts/job/registry/zookeeper/StateListener.java b/job-core/src/main/java/com/lts/job/zookeeper/StateListener.java similarity index 84% rename from job-core/src/main/java/com/lts/job/registry/zookeeper/StateListener.java rename to job-core/src/main/java/com/lts/job/zookeeper/StateListener.java index 9ea7c07f2..84aca7c88 100755 --- a/job-core/src/main/java/com/lts/job/registry/zookeeper/StateListener.java +++ b/job-core/src/main/java/com/lts/job/zookeeper/StateListener.java @@ -1,4 +1,4 @@ -package com.lts.job.registry.zookeeper; +package com.lts.job.zookeeper; /** * @author Robert HG (254963746@qq.com) on 7/8/14. diff --git a/job-core/src/main/java/com/lts/job/registry/zookeeper/ZookeeperClient.java b/job-core/src/main/java/com/lts/job/zookeeper/ZookeeperClient.java similarity index 86% rename from job-core/src/main/java/com/lts/job/registry/zookeeper/ZookeeperClient.java rename to job-core/src/main/java/com/lts/job/zookeeper/ZookeeperClient.java index a672e3489..06884612e 100755 --- a/job-core/src/main/java/com/lts/job/registry/zookeeper/ZookeeperClient.java +++ b/job-core/src/main/java/com/lts/job/zookeeper/ZookeeperClient.java @@ -1,4 +1,4 @@ -package com.lts.job.registry.zookeeper; +package com.lts.job.zookeeper; import java.util.List; @@ -11,12 +11,14 @@ public interface ZookeeperClient { String create(String path, Object data, boolean ephemeral, boolean sequential); - void delete(String path); + boolean delete(String path); boolean exists(String path); T getData(String path); + void setData(String path, Object data); + List getChildren(String path); List addChildListener(String path, ChildListener listener); diff --git a/job-core/src/main/java/com/lts/job/zookeeper/curator/CuratorZookeeperClient.java b/job-core/src/main/java/com/lts/job/zookeeper/curator/CuratorZookeeperClient.java new file mode 100644 index 000000000..ee3cd7d49 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/zookeeper/curator/CuratorZookeeperClient.java @@ -0,0 +1,237 @@ +package com.lts.job.zookeeper.curator; + +import com.lts.job.zookeeper.ChildListener; +import com.lts.job.zookeeper.StateListener; +import com.lts.job.zookeeper.serializer.SerializableSerializer; +import com.lts.job.zookeeper.serializer.ZkSerializer; +import com.lts.job.zookeeper.support.AbstractZookeeperClient; +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.framework.api.CuratorWatcher; +import com.netflix.curator.framework.state.ConnectionState; +import com.netflix.curator.framework.state.ConnectionStateListener; +import com.netflix.curator.retry.RetryNTimes; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +/** + * @author Robert HG (254963746@qq.com) on 5/16/15. + */ +public class CuratorZookeeperClient extends AbstractZookeeperClient { + + private final CuratorFramework client; + private final ZkSerializer zkSerializer; + + public CuratorZookeeperClient(String address) { + try { + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .connectString(address) + .retryPolicy(new RetryNTimes(Integer.MAX_VALUE, 1000)) + .connectionTimeoutMs(5000); + + client = builder.build(); + + client.getConnectionStateListenable().addListener(new ConnectionStateListener() { + public void stateChanged(CuratorFramework client, ConnectionState state) { + if (state == ConnectionState.LOST) { + CuratorZookeeperClient.this.stateChanged(StateListener.DISCONNECTED); + } else if (state == ConnectionState.CONNECTED) { + CuratorZookeeperClient.this.stateChanged(StateListener.CONNECTED); + } else if (state == ConnectionState.RECONNECTED) { + CuratorZookeeperClient.this.stateChanged(StateListener.RECONNECTED); + } + } + }); + + zkSerializer = new SerializableSerializer(); + + client.start(); + + } catch (IOException e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + protected String createPersistent(String path, boolean sequential) { + try { + if (sequential) { + return client.create().withMode(CreateMode.PERSISTENT_SEQUENTIAL).forPath(path); + } else { + return client.create().withMode(CreateMode.PERSISTENT).forPath(path); + } + } catch (KeeperException.NodeExistsException e) { + return path; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + protected String createPersistent(String path, Object data, boolean sequential) { + try { + if (sequential) { + byte[] zkDataBytes; + if (data instanceof Serializable) { + zkDataBytes = zkSerializer.serialize(data); + } else { + zkDataBytes = (byte[]) data; + } + return client.create().withMode(CreateMode.PERSISTENT_SEQUENTIAL).forPath(path, zkDataBytes); + } else { + return client.create().withMode(CreateMode.PERSISTENT).forPath(path); + } + } catch (KeeperException.NodeExistsException e) { + return path; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + protected String createEphemeral(String path, boolean sequential) { + try { + if (sequential) { + return client.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(path); + } else { + return client.create().withMode(CreateMode.EPHEMERAL).forPath(path); + } + } catch (KeeperException.NodeExistsException e) { + return path; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + protected String createEphemeral(String path, Object data, boolean sequential) { + try { + if (sequential) { + byte[] zkDataBytes; + if (data instanceof Serializable) { + zkDataBytes = zkSerializer.serialize(data); + } else { + zkDataBytes = (byte[]) data; + } + return client.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(path, zkDataBytes); + } else { + return client.create().withMode(CreateMode.EPHEMERAL).forPath(path); + } + } catch (KeeperException.NodeExistsException e) { + return path; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + protected CuratorWatcher createTargetChildListener(String path, ChildListener listener) { + return new CuratorWatcherImpl(listener); + } + + @Override + protected List addTargetChildListener(String path, CuratorWatcher listener) { + try { + return client.getChildren().usingWatcher(listener).forPath(path); + } catch (KeeperException.NoNodeException e) { + return null; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + protected void removeTargetChildListener(String path, CuratorWatcher listener) { + ((CuratorWatcherImpl) listener).unwatch(); + } + + @Override + public boolean delete(String path) { + try { + client.delete().forPath(path); + return true; + } catch (KeeperException.NoNodeException e) { + return true; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + public boolean exists(String path) { + try { + return client.checkExists().forPath(path) != null; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + public T getData(String path) { + try { + return (T) zkSerializer.deserialize(client.getData().forPath(path)); + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + public void setData(String path, Object data) { + byte[] zkDataBytes; + if (data instanceof Serializable) { + zkDataBytes = zkSerializer.serialize(data); + } else { + zkDataBytes = (byte[]) data; + } + try { + client.setData().forPath(path, zkDataBytes); + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + public List getChildren(String path) { + try { + return client.getChildren().forPath(path); + } catch (KeeperException.NoNodeException e) { + return null; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + @Override + public boolean isConnected() { + return client.getZookeeperClient().isConnected(); + } + + @Override + protected void doClose() { + client.close(); + } + + private class CuratorWatcherImpl implements CuratorWatcher { + + private volatile ChildListener listener; + + public CuratorWatcherImpl(ChildListener listener) { + this.listener = listener; + } + + public void unwatch() { + this.listener = null; + } + + public void process(WatchedEvent event) throws Exception { + if (listener != null) { + listener.childChanged(event.getPath(), client.getChildren().usingWatcher(this).forPath(event.getPath())); + } + } + } +} diff --git a/job-core/src/main/java/com/lts/job/zookeeper/serializer/SerializableSerializer.java b/job-core/src/main/java/com/lts/job/zookeeper/serializer/SerializableSerializer.java new file mode 100644 index 000000000..c1bf079b1 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/zookeeper/serializer/SerializableSerializer.java @@ -0,0 +1,40 @@ +package com.lts.job.zookeeper.serializer; + + +import java.io.*; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public class SerializableSerializer implements ZkSerializer { + @Override + public byte[] serialize(Object serializable) throws ZkMarshallingException { + try { + if (serializable == null) { + return null; + } + ByteArrayOutputStream byteArrayOS = new ByteArrayOutputStream(); + ObjectOutputStream stream = new ObjectOutputStream(byteArrayOS); + stream.writeObject(serializable); + stream.close(); + return byteArrayOS.toByteArray(); + } catch (IOException e) { + throw new ZkMarshallingException(e); + } + } + + @Override + public Object deserialize(byte[] bytes) throws ZkMarshallingException { + if (bytes == null) { + return null; + } + try { + ObjectInputStream inputStream = new ObjectInputStream(new ByteArrayInputStream(bytes)); + return inputStream.readObject(); + } catch (ClassNotFoundException e) { + throw new ZkMarshallingException("Unable to find object class.", e); + } catch (IOException e) { + throw new ZkMarshallingException(e); + } + } +} diff --git a/job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkMarshallingException.java b/job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkMarshallingException.java new file mode 100644 index 000000000..7c30d0fca --- /dev/null +++ b/job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkMarshallingException.java @@ -0,0 +1,27 @@ +package com.lts.job.zookeeper.serializer; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public class ZkMarshallingException extends RuntimeException { + + public ZkMarshallingException() { + super(); + } + + public ZkMarshallingException(String message) { + super(message); + } + + public ZkMarshallingException(String message, Throwable cause) { + super(message, cause); + } + + public ZkMarshallingException(Throwable cause) { + super(cause); + } + + protected ZkMarshallingException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkSerializer.java b/job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkSerializer.java new file mode 100644 index 000000000..139668ff6 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/zookeeper/serializer/ZkSerializer.java @@ -0,0 +1,11 @@ +package com.lts.job.zookeeper.serializer; + +/** + * @author Robert HG (254963746@qq.com) on 5/17/15. + */ +public interface ZkSerializer { + + public byte[] serialize(Object data) throws ZkMarshallingException; + + public Object deserialize(byte[] bytes) throws ZkMarshallingException; +} diff --git a/job-core/src/main/java/com/lts/job/registry/zookeeper/support/AbstractZookeeperClient.java b/job-core/src/main/java/com/lts/job/zookeeper/support/AbstractZookeeperClient.java similarity index 95% rename from job-core/src/main/java/com/lts/job/registry/zookeeper/support/AbstractZookeeperClient.java rename to job-core/src/main/java/com/lts/job/zookeeper/support/AbstractZookeeperClient.java index a10357eb8..48231436b 100755 --- a/job-core/src/main/java/com/lts/job/registry/zookeeper/support/AbstractZookeeperClient.java +++ b/job-core/src/main/java/com/lts/job/zookeeper/support/AbstractZookeeperClient.java @@ -1,9 +1,9 @@ -package com.lts.job.registry.zookeeper.support; +package com.lts.job.zookeeper.support; -import com.lts.job.registry.zookeeper.ChildListener; -import com.lts.job.registry.zookeeper.StateListener; -import com.lts.job.registry.zookeeper.ZookeeperClient; +import com.lts.job.zookeeper.ChildListener; +import com.lts.job.zookeeper.StateListener; +import com.lts.job.zookeeper.ZookeeperClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/job-core/src/main/java/com/lts/job/registry/zookeeper/zkclient/ZkClientZookeeperClient.java b/job-core/src/main/java/com/lts/job/zookeeper/zkclient/ZkClientZookeeperClient.java similarity index 91% rename from job-core/src/main/java/com/lts/job/registry/zookeeper/zkclient/ZkClientZookeeperClient.java rename to job-core/src/main/java/com/lts/job/zookeeper/zkclient/ZkClientZookeeperClient.java index 380759995..d26ad02c2 100755 --- a/job-core/src/main/java/com/lts/job/registry/zookeeper/zkclient/ZkClientZookeeperClient.java +++ b/job-core/src/main/java/com/lts/job/zookeeper/zkclient/ZkClientZookeeperClient.java @@ -1,8 +1,8 @@ -package com.lts.job.registry.zookeeper.zkclient; +package com.lts.job.zookeeper.zkclient; -import com.lts.job.registry.zookeeper.ChildListener; -import com.lts.job.registry.zookeeper.StateListener; -import com.lts.job.registry.zookeeper.support.AbstractZookeeperClient; +import com.lts.job.zookeeper.ChildListener; +import com.lts.job.zookeeper.StateListener; +import com.lts.job.zookeeper.support.AbstractZookeeperClient; import org.I0Itec.zkclient.IZkChildListener; import org.I0Itec.zkclient.IZkStateListener; import org.I0Itec.zkclient.ZkClient; @@ -27,6 +27,7 @@ public class ZkClientZookeeperClient extends AbstractZookeeperClient T getData(String path) { return zkClient.readData(path); } + @Override + public void setData(String path, Object data) { + zkClient.writeData(path, data); + } + @Override public List getChildren(String path) { try { diff --git a/job-core/src/test/java/com/lts/job/core/FileAccessorTest.java b/job-core/src/test/java/com/lts/job/core/FileAccessorTest.java deleted file mode 100644 index 8ae9a4377..000000000 --- a/job-core/src/test/java/com/lts/job/core/FileAccessorTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.lts.job.core; - -import com.lts.job.core.cluster.Node; -import com.lts.job.core.cluster.NodeType; -import com.lts.job.core.file.FileAccessor; -import com.lts.job.core.file.FileException; -import com.lts.job.core.file.Line; -import com.lts.job.core.util.JSONUtils; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author Robert HG (254963746@qq.com) on 8/14/14. - */ -public class FileAccessorTest { - - @Test - public void test(){ - - try { - FileAccessor fileAccessor = FileAccessor.create("/Users/hugui/Documents/JOB/QN_JOB.info"); - - fileAccessor.empty(); - - Node node = new Node(); - node.setPath("ddd中文测试"); - node.setNodeType(NodeType.CLIENT); - - fileAccessor.addOneLine(new Line(JSONUtils.toJSONString(node))); - fileAccessor.addOneLine(new Line(JSONUtils.toJSONString(node))); - fileAccessor.addOneLine(new Line(JSONUtils.toJSONString(node))); - - List lines = new ArrayList(); - for (int i = 0; i < 5; i++) { - node.setPath("中文测试" + i); - lines.add(new Line(JSONUtils.toJSONString(node))); - } - - fileAccessor.addLines(lines); - - lines = fileAccessor.readLines(); - for (int i = 0; i < lines.size(); i++) { - System.out.println(i + " : " + lines.get(i)); - } - - } catch (FileException e) { - e.printStackTrace(); - } - - - } - -} diff --git a/job-core/src/test/java/com/lts/job/registry/ZooTest.java b/job-core/src/test/java/com/lts/job/registry/ZooTest.java index 9646e664b..1d4e537be 100644 --- a/job-core/src/test/java/com/lts/job/registry/ZooTest.java +++ b/job-core/src/test/java/com/lts/job/registry/ZooTest.java @@ -1,8 +1,8 @@ package com.lts.job.registry; -import com.lts.job.registry.zookeeper.ChildListener; -import com.lts.job.registry.zookeeper.ZookeeperClient; -import com.lts.job.registry.zookeeper.zkclient.ZkClientZookeeperClient; +import com.lts.job.zookeeper.ChildListener; +import com.lts.job.zookeeper.ZookeeperClient; +import com.lts.job.zookeeper.zkclient.ZkClientZookeeperClient; import org.junit.Test; import java.io.IOException; diff --git a/job-example/pom.xml b/job-example/pom.xml index b8d8efb70..3650ab410 100644 --- a/job-example/pom.xml +++ b/job-example/pom.xml @@ -36,5 +36,10 @@ job-mongo-queue ${project.version} + + com.github.sgroschupf + zkclient + ${zkclient.version} + \ No newline at end of file diff --git a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java index 935440a2f..3c5864505 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java @@ -4,7 +4,7 @@ import com.lts.job.client.RetryJobClient; import com.lts.job.example.support.BaseJobClientTest; import com.lts.job.example.support.JobFinishedHandlerImpl; -import com.lts.job.example.support.MasterNodeChangeListenerImpl; +import com.lts.job.example.support.MasterChangeListenerImpl; import java.io.IOException; @@ -19,11 +19,11 @@ public static void main(String[] args) throws IOException { // final JobClient jobClient = new JobClient(); jobClient.setNodeGroup("test_jobClient"); // jobClient.setClusterName("lts"); - jobClient.setZookeeperAddress("localhost:2181"); + jobClient.setRegistryAddress("zookeeper://localhost:2181"); // 任务重试保存地址，默认用户目录下 // jobClient.setJobInfoSavePath(Constants.USER_HOME); jobClient.setJobFinishedHandler(new JobFinishedHandlerImpl()); - jobClient.addMasterNodeChangeListener(new MasterNodeChangeListenerImpl()); + jobClient.addMasterChangeListener(new MasterChangeListenerImpl()); jobClient.start(); JobClientTest jobClientTest = new JobClientTest(); diff --git a/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java b/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java index 6687f963f..3a9b939cb 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java @@ -1,6 +1,6 @@ package com.lts.job.example.api; -import com.lts.job.example.support.MasterNodeChangeListenerImpl; +import com.lts.job.example.support.MasterChangeListenerImpl; import com.lts.job.queue.mongo.MongoJobFeedbackQueue; import com.lts.job.queue.mongo.MongoJobLogger; import com.lts.job.queue.mongo.MongoJobQueue; @@ -17,11 +17,11 @@ public static void main(String[] args) { final JobTracker jobTracker = new JobTracker(); // 节点信息配置 - jobTracker.setZookeeperAddress("localhost:2181"); + jobTracker.setRegistryAddress("zookeeper://localhost:2181"); jobTracker.setListenPort(35002); // 默认 35001 // jobTracker.setClusterName("lts"); - jobTracker.addMasterNodeChangeListener(new MasterNodeChangeListenerImpl()); + jobTracker.addMasterChangeListener(new MasterChangeListenerImpl()); // mongo 配置 Config config = new Config(); diff --git a/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java b/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java index e8230c79b..f8f3def6d 100644 --- a/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java @@ -1,6 +1,6 @@ package com.lts.job.example.api; -import com.lts.job.example.support.MasterNodeChangeListenerImpl; +import com.lts.job.example.support.MasterChangeListenerImpl; import com.lts.job.example.support.TestJobRunner; import com.lts.job.task.tracker.TaskTracker; @@ -13,12 +13,12 @@ public class TaskTrackerTest { public static void main(String[] args) { final TaskTracker taskTracker = new TaskTracker(); taskTracker.setJobRunnerClass(TestJobRunner.class); - taskTracker.setZookeeperAddress("localhost:2181"); + taskTracker.setRegistryAddress("zookeeper://localhost:2181"); taskTracker.setNodeGroup("test_trade_TaskTracker"); // taskTracker.setClusterName("lts"); taskTracker.setWorkThreads(20); // taskTracker.setJobInfoSavePath(Constants.USER_HOME); - taskTracker.addMasterNodeChangeListener(new MasterNodeChangeListenerImpl()); + taskTracker.addMasterChangeListener(new MasterChangeListenerImpl()); // taskTracker.setBizLoggerLevel(Level.INFO); // 业务日志级别 taskTracker.start(); diff --git a/job-example/src/main/java/com/lts/job/example/support/MasterNodeChangeListenerImpl.java b/job-example/src/main/java/com/lts/job/example/support/MasterChangeListenerImpl.java similarity index 85% rename from job-example/src/main/java/com/lts/job/example/support/MasterNodeChangeListenerImpl.java rename to job-example/src/main/java/com/lts/job/example/support/MasterChangeListenerImpl.java index 014b32aa8..9a1922fa1 100644 --- a/job-example/src/main/java/com/lts/job/example/support/MasterNodeChangeListenerImpl.java +++ b/job-example/src/main/java/com/lts/job/example/support/MasterChangeListenerImpl.java @@ -1,13 +1,13 @@ package com.lts.job.example.support; import com.lts.job.core.cluster.Node; -import com.lts.job.core.listener.MasterNodeChangeListener; +import com.lts.job.core.listener.MasterChangeListener; import com.lts.job.core.util.StringUtils; /** * @author Robert HG (254963746@qq.com) on 3/6/15. */ -public class MasterNodeChangeListenerImpl implements MasterNodeChangeListener { +public class MasterChangeListenerImpl implements MasterChangeListener { /** * master 为 master节点 diff --git a/job-example/src/main/resources/lts-spring-job-client.xml b/job-example/src/main/resources/lts-spring-job-client.xml index f1df8c779..9e5e03ac6 100644 --- a/job-example/src/main/resources/lts-spring-job-client.xml +++ b/job-example/src/main/resources/lts-spring-job-client.xml @@ -9,13 +9,13 @@ - + - + - + diff --git a/job-example/src/main/resources/lts-spring-job-tacker.xml b/job-example/src/main/resources/lts-spring-job-tacker.xml index c223fcfa3..376f5cbe3 100644 --- a/job-example/src/main/resources/lts-spring-job-tacker.xml +++ b/job-example/src/main/resources/lts-spring-job-tacker.xml @@ -29,13 +29,13 @@ - + - + - + diff --git a/job-example/src/main/resources/lts-spring-job-tasktracker.xml b/job-example/src/main/resources/lts-spring-job-tasktracker.xml index 9bb7f4e83..20c7a960f 100644 --- a/job-example/src/main/resources/lts-spring-job-tasktracker.xml +++ b/job-example/src/main/resources/lts-spring-job-tasktracker.xml @@ -8,12 +8,12 @@ - + - + - + diff --git a/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobClientFactoryBean.java b/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobClientFactoryBean.java index 3b9282f75..9b88a86d0 100644 --- a/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobClientFactoryBean.java +++ b/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobClientFactoryBean.java @@ -3,7 +3,7 @@ import com.lts.job.client.JobClient; import com.lts.job.client.RetryJobClient; import com.lts.job.client.support.JobFinishedHandler; -import com.lts.job.core.listener.MasterNodeChangeListener; +import com.lts.job.core.listener.MasterChangeListener; import com.lts.job.core.util.Assert; import com.lts.job.core.util.StringUtils; import org.springframework.beans.factory.DisposableBean; @@ -32,7 +32,7 @@ public class JobClientFactoryBean implements FactoryBean, Initializin /** * zookeeper地址 */ - private String zookeeperAddress; + private String registryAddress; /** * 任务完成处理器 */ @@ -44,7 +44,7 @@ public class JobClientFactoryBean implements FactoryBean, Initializin /** * master节点变化监听器 */ - private MasterNodeChangeListener[] masterNodeChangeListeners; + private MasterChangeListener[] masterChangeListeners; public void setClientType(String type) { if (type == null) { @@ -88,7 +88,7 @@ public void checkProperties() { clientType = JobClientType.NORMAL; } Assert.hasText(nodeGroup, "nodeGroup必须设值!"); - Assert.hasText(zookeeperAddress, "zookeeperAddress必须设值!"); + Assert.hasText(registryAddress, "registryAddress必须设值!"); } @Override @@ -109,11 +109,11 @@ public void afterPropertiesSet() throws Exception { if (jobFinishedHandler != null) { jobClient.setJobFinishedHandler(jobFinishedHandler); } - jobClient.setZookeeperAddress(zookeeperAddress); + jobClient.setRegistryAddress(registryAddress); - if (masterNodeChangeListeners != null) { - for (MasterNodeChangeListener masterNodeChangeListener : masterNodeChangeListeners) { - jobClient.addMasterNodeChangeListener(masterNodeChangeListener); + if (masterChangeListeners != null) { + for (MasterChangeListener masterChangeListener : masterChangeListeners) { + jobClient.addMasterChangeListener(masterChangeListener); } } } @@ -133,8 +133,8 @@ public void setNodeGroup(String nodeGroup) { this.nodeGroup = nodeGroup; } - public void setZookeeperAddress(String zookeeperAddress) { - this.zookeeperAddress = zookeeperAddress; + public void setRegistryAddress(String registryAddress) { + this.registryAddress = registryAddress; } public void setJobFinishedHandler(JobFinishedHandler jobFinishedHandler) { @@ -145,8 +145,8 @@ public void setJobInfoSavePath(String jobInfoSavePath) { this.jobInfoSavePath = jobInfoSavePath; } - public void setMasterNodeChangeListeners(MasterNodeChangeListener[] masterNodeChangeListeners) { - this.masterNodeChangeListeners = masterNodeChangeListeners; + public void setMasterChangeListeners(MasterChangeListener[] masterChangeListeners) { + this.masterChangeListeners = masterChangeListeners; } } diff --git a/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobTrackerFactoryBean.java b/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobTrackerFactoryBean.java index b107fabc2..697e33d70 100644 --- a/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobTrackerFactoryBean.java +++ b/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/JobTrackerFactoryBean.java @@ -1,6 +1,6 @@ package com.lts.job.spring; -import com.lts.job.core.listener.MasterNodeChangeListener; +import com.lts.job.core.listener.MasterChangeListener; import com.lts.job.core.util.Assert; import com.lts.job.core.util.StringUtils; import com.lts.job.tracker.JobTracker; @@ -31,11 +31,11 @@ public class JobTrackerFactoryBean implements FactoryBean, Initializ /** * zookeeper地址 */ - private String zookeeperAddress; + private String registryAddress; /** * master节点变化监听器 */ - private MasterNodeChangeListener[] masterNodeChangeListeners; + private MasterChangeListener[] masterChangeListeners; private JobLogger jobLogger; @@ -67,7 +67,7 @@ public void destroy() throws Exception { } public void checkProperties() { - Assert.hasText(zookeeperAddress, "zookeeperAddress必须设值!"); + Assert.hasText(registryAddress, "registryAddress必须设值!"); if (listenPort != null && listenPort <= 0) { listenPort = null; } @@ -83,13 +83,13 @@ public void afterPropertiesSet() throws Exception { if (StringUtils.hasText(clusterName)) { jobTracker.setClusterName(clusterName); } - jobTracker.setZookeeperAddress(zookeeperAddress); + jobTracker.setRegistryAddress(registryAddress); if (listenPort != null) { jobTracker.setListenPort(listenPort); } - if (masterNodeChangeListeners != null) { - for (MasterNodeChangeListener masterNodeChangeListener : masterNodeChangeListeners) { - jobTracker.addMasterNodeChangeListener(masterNodeChangeListener); + if (masterChangeListeners != null) { + for (MasterChangeListener masterChangeListener : masterChangeListeners) { + jobTracker.addMasterChangeListener(masterChangeListener); } } jobTracker.setJobLogger(jobLogger); @@ -112,12 +112,12 @@ public void setListenPort(Integer listenPort) { this.listenPort = listenPort; } - public void setZookeeperAddress(String zookeeperAddress) { - this.zookeeperAddress = zookeeperAddress; + public void setRegistryAddress(String registryAddress) { + this.registryAddress = registryAddress; } - public void setMasterNodeChangeListeners(MasterNodeChangeListener[] masterNodeChangeListeners) { - this.masterNodeChangeListeners = masterNodeChangeListeners; + public void setMasterChangeListeners(MasterChangeListener[] masterChangeListeners) { + this.masterChangeListeners = masterChangeListeners; } public void setJobLogger(JobLogger jobLogger) { diff --git a/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/TaskTrackerFactoryBean.java b/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/TaskTrackerFactoryBean.java index fb02816b5..922bf3f7a 100644 --- a/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/TaskTrackerFactoryBean.java +++ b/job-extensions/job-ext-spring/src/main/java/com/lts/job/spring/TaskTrackerFactoryBean.java @@ -1,6 +1,6 @@ package com.lts.job.spring; -import com.lts.job.core.listener.MasterNodeChangeListener; +import com.lts.job.core.listener.MasterChangeListener; import com.lts.job.core.util.Assert; import com.lts.job.core.util.StringUtils; import com.lts.job.task.tracker.TaskTracker; @@ -37,7 +37,7 @@ public class TaskTrackerFactoryBean implements FactoryBean, Applica /** * zookeeper地址 */ - private String zookeeperAddress; + private String registryAddress; /** * 提交失败任务存储路径 , 默认用户木邻居 */ @@ -53,7 +53,7 @@ public class TaskTrackerFactoryBean implements FactoryBean, Applica /** * master节点变化监听器 */ - private MasterNodeChangeListener[] masterNodeChangeListeners; + private MasterChangeListener[] masterChangeListeners; @Override public TaskTracker getObject() throws Exception { @@ -80,7 +80,7 @@ public void destroy() throws Exception { public void checkProperties() { Assert.hasText(nodeGroup, "nodeGroup必须设值!"); - Assert.hasText(zookeeperAddress, "zookeeperAddress必须设值!"); + Assert.hasText(registryAddress, "registryAddress必须设值!"); Assert.isTrue(workThreads > 0, "workThreads必须大于0!"); Assert.notNull(jobRunnerClass, "jobRunnerClass不能为空"); Assert.isAssignable(JobRunner.class, jobRunnerClass, @@ -100,14 +100,14 @@ public void afterPropertiesSet() throws Exception { taskTracker.setJobInfoSavePath(jobInfoSavePath); taskTracker.setWorkThreads(workThreads); taskTracker.setNodeGroup(nodeGroup); - taskTracker.setZookeeperAddress(zookeeperAddress); + taskTracker.setRegistryAddress(registryAddress); taskTracker.setJobRunnerClass(jobRunnerClass); registerRunnerBeanDefinition(); - if (masterNodeChangeListeners != null) { - for (MasterNodeChangeListener masterNodeChangeListener : masterNodeChangeListeners) { - taskTracker.addMasterNodeChangeListener(masterNodeChangeListener); + if (masterChangeListeners != null) { + for (MasterChangeListener masterChangeListener : masterChangeListeners) { + taskTracker.addMasterChangeListener(masterChangeListener); } } } @@ -142,16 +142,16 @@ public void setNodeGroup(String nodeGroup) { this.nodeGroup = nodeGroup; } - public void setZookeeperAddress(String zookeeperAddress) { - this.zookeeperAddress = zookeeperAddress; + public void setRegistryAddress(String registryAddress) { + this.registryAddress = registryAddress; } public void setJobInfoSavePath(String jobInfoSavePath) { this.jobInfoSavePath = jobInfoSavePath; } - public void setMasterNodeChangeListeners(MasterNodeChangeListener[] masterNodeChangeListeners) { - this.masterNodeChangeListeners = masterNodeChangeListeners; + public void setMasterChangeListeners(MasterChangeListener[] masterChangeListeners) { + this.masterChangeListeners = masterChangeListeners; } public void setJobRunnerClass(Class jobRunnerClass) { diff --git a/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java b/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java index 20881646f..1d796ff13 100644 --- a/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java +++ b/job-task-tracker/src/test/java/com/lts/job/task/tracker/ThreadPoolDynamicTest.java @@ -5,7 +5,7 @@ import java.util.concurrent.TimeUnit; /** - * Created by hugui on 5/11/15. + * @author Robert HG (254963746@qq.com) on 5/11/15. */ public class ThreadPoolDynamicTest { diff --git a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java index 7184f84e5..b63884719 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java @@ -26,10 +26,8 @@ public class JobTracker extends AbstractServerNode getChannels(String nodeGroup, NodeType nodeType) { /** * 根据 节点唯一编号得到 channel + * * @param nodeGroup * @param nodeType * @param identity diff --git a/job-tracker/src/main/java/com/lts/job/tracker/logger/domain/BizLogPo.java b/job-tracker/src/main/java/com/lts/job/tracker/logger/domain/BizLogPo.java index 98cd3a4c1..7977b7ed6 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/logger/domain/BizLogPo.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/logger/domain/BizLogPo.java @@ -3,7 +3,7 @@ import com.lts.job.core.constant.Level; /** - * Created by hugui on 3/30/15. + * @author Robert HG (254963746@qq.com) on 3/30/15. */ public class BizLogPo { diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java b/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java index 148767f46..8784fa2b4 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java @@ -34,7 +34,8 @@ public TaskTrackerManager(ChannelManager channelManager) { */ public void addNode(Node node) { // channel 可能为 null - ChannelWrapper channel = channelManager.getChannel(node.getGroup(), node.getNodeType(), node.getIdentity()); + ChannelWrapper channel = channelManager.getChannel(node.getGroup(), + node.getNodeType(), node.getIdentity()); ConcurrentHashSet taskTrackerNodes = NODE_MAP.get(node.getGroup()); synchronized (NODE_MAP) { @@ -44,7 +45,8 @@ public void addNode(Node node) { } } - TaskTrackerNode taskTrackerNode = new TaskTrackerNode(node.getGroup(), node.getThreads(), node.getIdentity(), channel); + TaskTrackerNode taskTrackerNode = new TaskTrackerNode(node.getGroup(), + node.getThreads(), node.getIdentity(), channel); LOGGER.info("添加TaskTracker节点:{}", taskTrackerNode); taskTrackerNodes.add(taskTrackerNode); } @@ -74,14 +76,15 @@ public TaskTrackerNode getTaskTrackerNode(String nodeGroup, String identity) { if (taskTrackerNode.getIdentity().equals(identity)) { if (taskTrackerNode.getChannel() == null || taskTrackerNode.getChannel().isClosed()) { // 如果 channel 已经关闭, 更新channel, 如果没有channel, 略过 - ChannelWrapper channel = channelManager.getChannel(taskTrackerNode.getNodeGroup(), NodeType.TASK_TRACKER, taskTrackerNode.getIdentity()); + ChannelWrapper channel = channelManager.getChannel( + taskTrackerNode.getNodeGroup(), NodeType.TASK_TRACKER, taskTrackerNode.getIdentity()); if (channel != null) { // 更新channel taskTrackerNode.setChannel(channel); LOGGER.info("更新节点channel , taskTackerNode={}", taskTrackerNode); return taskTrackerNode; } - }else{ + } else { // 只有当channel正常的时候才返回 return taskTrackerNode; } @@ -98,13 +101,18 @@ public TaskTrackerNode getTaskTrackerNode(String nodeGroup, String identity) { * @param availableThreads * @param timestamp 时间戳, 只有当 时间戳大于上次更新的时间 才更新可用线程数 */ - public void updateTaskTrackerAvailableThreads(String nodeGroup, String identity, Integer availableThreads, Long timestamp) { + public void updateTaskTrackerAvailableThreads( + String nodeGroup, + String identity, + Integer availableThreads, + Long timestamp) { ConcurrentHashSet taskTrackerNodes = NODE_MAP.get(nodeGroup); if (taskTrackerNodes != null && taskTrackerNodes.size() != 0) { for (TaskTrackerNode trackerNode : taskTrackerNodes) { - if (trackerNode.getIdentity().equals(identity) && trackerNode.getTimestamp() <= timestamp) { + if (trackerNode.getIdentity().equals(identity) + && trackerNode.getTimestamp() <= timestamp) { trackerNode.setAvailableThread(availableThreads); trackerNode.setTimestamp(timestamp); if (LOGGER.isDebugEnabled()) { diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java b/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java index 25305764b..f18d061b4 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java @@ -3,9 +3,8 @@ import com.lts.job.core.cluster.Node; import com.lts.job.core.cluster.NodeType; import com.lts.job.core.listener.NodeChangeListener; +import com.lts.job.core.util.CollectionUtils; import com.lts.job.tracker.domain.JobTrackerApplication; -import com.lts.job.tracker.support.JobClientManager; -import com.lts.job.tracker.support.TaskTrackerManager; import java.util.List; @@ -19,36 +18,34 @@ public class JobNodeChangeListener implements NodeChangeListener { public JobNodeChangeListener(JobTrackerApplication application) { this.application = application; - this.jobClientManager = application.getJobClientManager(); - this.taskTrackerManager = application.getTaskTrackerManager(); } - private TaskTrackerManager taskTrackerManager; - private JobClientManager jobClientManager; - @Override - public void addNode(Node node) { - if (node.getNodeType().equals(NodeType.TASK_TRACKER)) { - taskTrackerManager.addNode(node); - } else if (node.getNodeType().equals(NodeType.CLIENT)) { - jobClientManager.addNode(node); + public void addNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; } - } - - @Override - public void removeNode(Node node) { - if (node.getNodeType().equals(NodeType.TASK_TRACKER)) { - taskTrackerManager.removeNode(node); - application.getDeadJobChecker().fixedDeadLock(node); - } else if (node.getNodeType().equals(NodeType.CLIENT)) { - jobClientManager.removeNode(node); + for (Node node : nodes) { + if (node.getNodeType().equals(NodeType.TASK_TRACKER)) { + application.getTaskTrackerManager().addNode(node); + } else if (node.getNodeType().equals(NodeType.CLIENT)) { + application.getJobClientManager().addNode(node); + } } } @Override - public void addNodes(List nodes) { + public void removeNodes(List nodes) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } for (Node node : nodes) { - addNode(node); + if (node.getNodeType().equals(NodeType.TASK_TRACKER)) { + application.getTaskTrackerManager().removeNode(node); + application.getDeadJobChecker().fixedDeadLock(node); + } else if (node.getNodeType().equals(NodeType.CLIENT)) { + application.getJobClientManager().removeNode(node); + } } } } diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobTrackerMasterChangeListener.java b/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobTrackerMasterChangeListener.java index 0d3b11c94..69096f3d2 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobTrackerMasterChangeListener.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobTrackerMasterChangeListener.java @@ -1,7 +1,7 @@ package com.lts.job.tracker.support.listener; import com.lts.job.core.cluster.Node; -import com.lts.job.core.listener.MasterNodeChangeListener; +import com.lts.job.core.listener.MasterChangeListener; import com.lts.job.tracker.domain.JobTrackerApplication; import com.lts.job.tracker.support.checker.DeadJobChecker; import com.lts.job.tracker.support.checker.FeedbackJobSendChecker; @@ -10,7 +10,7 @@ * @author Robert HG (254963746@qq.com) on 8/24/14. * JobTracker master 节点变化之后 */ -public class JobTrackerMasterChangeListener implements MasterNodeChangeListener { +public class JobTrackerMasterChangeListener implements MasterChangeListener { private JobTrackerApplication application; private DeadJobChecker deadJobChecker; diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/policy/OldDataDeletePolicy.java b/job-tracker/src/main/java/com/lts/job/tracker/support/policy/OldDataDeletePolicy.java index 1e033b33e..620442cfd 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/policy/OldDataDeletePolicy.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/policy/OldDataDeletePolicy.java @@ -5,7 +5,7 @@ import com.lts.job.tracker.support.OldDataHandler; /** - * Created by hugui on 3/30/15. + * @author Robert HG (254963746@qq.com) on 3/30/15. */ public class OldDataDeletePolicy implements OldDataHandler { diff --git a/pom.xml b/pom.xml index 9daa5de0e..cf16e8969 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,15 @@ job-admin + + 2.1.0 + 1.1.41 + 1.8 + 1.1.10 + 0.1 + 3.4.5 + + io.netty @@ -33,7 +42,7 @@ org.apache.zookeeper zookeeper - 3.4.5 + ${zk.version} org.jboss.netty @@ -48,12 +57,29 @@ com.github.sgroschupf zkclient - 0.1 + ${zkclient.version} + provided + + + com.netflix.curator + curator-framework + ${curator.version} + provided + + + redis.clients + jedis + ${jedis.version} + + + org.fusesource.leveldbjni + leveldbjni-all + ${leveldbjni.version} com.alibaba fastjson - 1.1.41 + ${fastjson.version} org.slf4j @@ -65,6 +91,7 @@ slf4j-log4j12 1.6.1 + From 0ed2c432147887cceed3bb104c96c6fa42415b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E8=B4=B5?= Date: Mon, 18 May 2015 17:58:28 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=8F=82=E8=80=83=20dubbo=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=20spi=20=E6=89=A9=E5=B1=95=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 +- .../java/com/lts/job/core/Application.java | 8 +- .../lts/job/core/cluster/AbstractJobNode.java | 11 +- .../java/com/lts/job/core/cluster/Config.java | 251 ++++++ .../com/lts/job/core/compiler/Compiler.java | 21 + .../compiler/support/AbstractCompiler.java | 69 ++ .../compiler/support/AdaptiveCompiler.java | 48 ++ .../job/core/compiler/support/ClassUtils.java | 413 ++++++++++ .../compiler/support/JavassistCompiler.java | 119 +++ .../core/compiler/support/JdkCompiler.java | 268 +++++++ .../com/lts/job/core/constant/Constants.java | 3 + .../lts/job/core/domain/JobNodeConfig.java | 142 ---- .../com/lts/job/core/extension/Activate.java | 78 ++ .../com/lts/job/core/extension/Adaptive.java | 53 ++ .../job/core/extension/ExtensionFactory.java | 34 + .../job/core/extension/ExtensionLoader.java | 735 ++++++++++++++++++ .../java/com/lts/job/core/extension/SPI.java | 33 + .../factory/AdaptiveExtensionFactory.java | 55 ++ .../factory/SpiExtensionFactory.java | 39 + .../core/factory/JobNodeConfigFactory.java | 8 +- .../com/lts/job/core/factory/NodeFactory.java | 4 +- .../job/core/listener/SelfChangeListener.java | 4 +- .../com/lts/job/core/util/ClassHelper.java | 188 +++++ .../java/com/lts/job/core/util/Holder.java | 18 + .../com/lts/job/core/util/StringUtils.java | 42 + .../com.lts.job.core.compiler.Compiler | 3 + ...om.lts.job.core.extension.ExtensionFactory | 2 + .../java/com/lts/job/core/FileLockTest.java | 2 +- .../java/com/lts/job/core/NetUtilsTest.java | 2 +- .../java/com/lts/job/core/spi/MainTest.java | 23 + .../com/lts/job/core/spi/TestService.java | 16 + .../com/lts/job/core/spi/TestServiceImpl.java | 14 + .../lts/job/core/spi/TestServiceImpl2.java | 14 + .../lts/com.lts.job.core.spi.TestService | 2 + .../lts/job/example/api/JobClientTest.java | 2 +- .../lts/job/example/api/JobTrackerTest.java | 4 +- .../lts/job/example/api/TaskTrackerTest.java | 2 +- .../main/resources/lts-spring-job-client.xml | 2 +- .../main/resources/lts-spring-job-tacker.xml | 4 +- .../resources/lts-spring-job-tasktracker.xml | 2 +- pom.xml | 8 +- 41 files changed, 2587 insertions(+), 175 deletions(-) create mode 100644 job-core/src/main/java/com/lts/job/core/cluster/Config.java create mode 100644 job-core/src/main/java/com/lts/job/core/compiler/Compiler.java create mode 100755 job-core/src/main/java/com/lts/job/core/compiler/support/AbstractCompiler.java create mode 100755 job-core/src/main/java/com/lts/job/core/compiler/support/AdaptiveCompiler.java create mode 100755 job-core/src/main/java/com/lts/job/core/compiler/support/ClassUtils.java create mode 100755 job-core/src/main/java/com/lts/job/core/compiler/support/JavassistCompiler.java create mode 100755 job-core/src/main/java/com/lts/job/core/compiler/support/JdkCompiler.java delete mode 100644 job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/Activate.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/Adaptive.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/ExtensionFactory.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/ExtensionLoader.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/SPI.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/factory/AdaptiveExtensionFactory.java create mode 100644 job-core/src/main/java/com/lts/job/core/extension/factory/SpiExtensionFactory.java create mode 100644 job-core/src/main/java/com/lts/job/core/util/ClassHelper.java create mode 100644 job-core/src/main/java/com/lts/job/core/util/Holder.java create mode 100644 job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.compiler.Compiler create mode 100755 job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.extension.ExtensionFactory create mode 100644 job-core/src/test/java/com/lts/job/core/spi/MainTest.java create mode 100644 job-core/src/test/java/com/lts/job/core/spi/TestService.java create mode 100644 job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl.java create mode 100644 job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl2.java create mode 100644 job-core/src/test/resources/META-INF/lts/com.lts.job.core.spi.TestService diff --git a/README.md b/README.md index d850f7f4a..90309495e 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) ```java final JobTracker jobTracker = new JobTracker(); // 节点信息配置 - jobTracker.setRegistryAddress("zookeeper://localhost:2181"); + jobTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); // jobTracker.setListenPort(35001); // 默认 35001 // jobTracker.setClusterName("lts"); // mongo 配置 Config config = new Config(); - config.setAddresses(new String[]{"localhost:27017"}); + config.setAddresses(new String[]{"127.0.0.1:27017"}); config.setUsername("lts"); config.setPassword("lts"); config.setDbName("job"); @@ -83,7 +83,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - localhost:27017 + 127.0.0.1:27017 @@ -93,7 +93,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - + @@ -108,7 +108,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) TaskTracker taskTracker = new TaskTracker(); taskTracker.setJobRunnerClass(TestJobRunner.class); // jobClient.setClusterName("lts"); - taskTracker.setRegistryAddress("zookeeper://localhost:2181"); + taskTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); taskTracker.setNodeGroup("test_trade_TaskTracker"); taskTracker.setWorkThreads(20); taskTracker.start(); @@ -136,7 +136,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - + @@ -153,7 +153,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) // JobClient jobClient = new JobClient(); jobClient.setNodeGroup("test_JobClient"); // jobClient.setClusterName("lts"); - jobClient.setRegistryAddress("zookeeper://localhost:2181"); + jobClient.setRegistryAddress("zookeeper://127.0.0.1:2181"); jobClient.start(); // 提交任务 @@ -170,7 +170,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule) - + diff --git a/job-core/src/main/java/com/lts/job/core/Application.java b/job-core/src/main/java/com/lts/job/core/Application.java index 0136a7430..07ccce091 100644 --- a/job-core/src/main/java/com/lts/job/core/Application.java +++ b/job-core/src/main/java/com/lts/job/core/Application.java @@ -2,7 +2,7 @@ import com.lts.job.core.cluster.MasterElector; import com.lts.job.core.cluster.SubscribedNodeManager; -import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.core.cluster.Config; import com.lts.job.core.protocol.command.CommandBodyWrapper; import com.lts.job.ec.EventCenter; @@ -13,7 +13,7 @@ public abstract class Application { // 节点配置信息 - private JobNodeConfig config; + private Config config; // 节点管理 private SubscribedNodeManager subscribedNodeManager; // master节点选举者 @@ -31,11 +31,11 @@ public void setCommandBodyWrapper(CommandBodyWrapper commandBodyWrapper) { this.commandBodyWrapper = commandBodyWrapper; } - public JobNodeConfig getConfig() { + public Config getConfig() { return config; } - public void setConfig(JobNodeConfig config) { + public void setConfig(Config config) { this.config = config; } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java index dae7d92a2..c5000a355 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java @@ -2,7 +2,6 @@ import com.lts.job.core.Application; import com.lts.job.core.constant.EcTopic; -import com.lts.job.core.domain.JobNodeConfig; import com.lts.job.core.factory.JobNodeConfigFactory; import com.lts.job.core.factory.NodeFactory; import com.lts.job.core.listener.MasterChangeListener; @@ -33,7 +32,7 @@ public abstract class AbstractJobNode i protected Registry registry; protected T node; - protected JobNodeConfig config; + protected Config config; protected App application; private List nodeChangeListeners; @@ -50,6 +49,9 @@ public AbstractJobNode() { final public void start() { try { + // 初始化配置 + initConfig(); + node = NodeFactory.create(getNodeClass(), config); config.setNodeType(node.getNodeType()); @@ -147,6 +149,11 @@ public void notify(NotifyEvent event, List nodes) { }); } + protected void initConfig() { + // do nothing 让子类去实现 + + } + protected abstract void innerStart(); protected abstract void innerStop(); diff --git a/job-core/src/main/java/com/lts/job/core/cluster/Config.java b/job-core/src/main/java/com/lts/job/core/cluster/Config.java new file mode 100644 index 000000000..8e4a9fdb5 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/cluster/Config.java @@ -0,0 +1,251 @@ +package com.lts.job.core.cluster; + +import com.lts.job.core.compiler.support.AdaptiveCompiler; +import com.lts.job.core.constant.Constants; +import com.lts.job.core.util.JSONUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author Robert HG (254963746@qq.com) on 8/20/14. + * 任务节点配置 + */ +public class Config { + + // 节点是否可用 + private boolean available = true; + // 应用节点组 + private String nodeGroup; + // 唯一标识 + private String identity; + // 工作线程 + private int workThreads; + // 节点类型 + private NodeType nodeType; + // 注册中心 地址 + private String registryAddress; + // 远程连接超时时间 + private int invokeTimeoutMillis; + // 监听端口 + private int listenPort; + // 任务信息存储路径(譬如TaskTracker反馈任务信息给JobTracker, JobTracker down掉了, 那么存储下来等待JobTracker可用时再发送) + private String jobInfoSavePath; + // 集群名字 + private String clusterName; + // java编译器 + private String compiler; + + private volatile transient Map numbers; + + private final Map parameters = new HashMap(); + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getNodeGroup() { + return nodeGroup; + } + + public void setNodeGroup(String nodeGroup) { + this.nodeGroup = nodeGroup; + } + + public String getIdentity() { + return identity; + } + + public void setIdentity(String identity) { + this.identity = identity; + } + + public int getWorkThreads() { + return workThreads; + } + + public void setWorkThreads(int workThreads) { + this.workThreads = workThreads; + } + + public NodeType getNodeType() { + return nodeType; + } + + public void setNodeType(NodeType nodeType) { + this.nodeType = nodeType; + } + + public String getRegistryAddress() { + return registryAddress; + } + + public void setRegistryAddress(String registryAddress) { + this.registryAddress = registryAddress; + } + + public int getInvokeTimeoutMillis() { + return invokeTimeoutMillis; + } + + public void setInvokeTimeoutMillis(int invokeTimeoutMillis) { + this.invokeTimeoutMillis = invokeTimeoutMillis; + } + + public int getListenPort() { + return listenPort; + } + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + public String getJobInfoSavePath() { + return jobInfoSavePath; + } + + public void setJobInfoSavePath(String jobInfoSavePath) { + this.jobInfoSavePath = jobInfoSavePath + "/.lts"; + } + + public String getFilePath() { + return jobInfoSavePath + "/" + nodeType + "/" + nodeGroup + "/"; + } + + public boolean isAvailable() { + return available; + } + + public void setAvailable(boolean available) { + this.available = available; + } + + public void setParameter(String key, String value) { + parameters.put(key, value); + } + + public String getParameter(String key) { + return parameters.get(key); + } + + public String getParameter(String key, String defaultValue) { + String value = parameters.get(key); + if (value == null) { + return defaultValue; + } + return value; + } + private Map getNumbers() { + if (numbers == null) { // 允许并发重复创建 + numbers = new ConcurrentHashMap(); + } + return numbers; + } + public int getParameter(String key, int defaultValue) { + Number n = getNumbers().get(key); + if (n != null) { + return n.intValue(); + } + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + int i = Integer.parseInt(value); + getNumbers().put(key, i); + return i; + } + public String[] getParameter(String key, String[] defaultValue) { + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + return Constants.COMMA_SPLIT_PATTERN.split(value); + } + public double getParameter(String key, double defaultValue) { + Number n = getNumbers().get(key); + if (n != null) { + return n.doubleValue(); + } + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + double d = Double.parseDouble(value); + getNumbers().put(key, d); + return d; + } + public float getParameter(String key, float defaultValue) { + Number n = getNumbers().get(key); + if (n != null) { + return n.floatValue(); + } + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + float f = Float.parseFloat(value); + getNumbers().put(key, f); + return f; + } + + public long getParameter(String key, long defaultValue) { + Number n = getNumbers().get(key); + if (n != null) { + return n.longValue(); + } + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + long l = Long.parseLong(value); + getNumbers().put(key, l); + return l; + } + + public short getParameter(String key, short defaultValue) { + Number n = getNumbers().get(key); + if (n != null) { + return n.shortValue(); + } + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + short s = Short.parseShort(value); + getNumbers().put(key, s); + return s; + } + + public byte getParameter(String key, byte defaultValue) { + Number n = getNumbers().get(key); + if (n != null) { + return n.byteValue(); + } + String value = getParameter(key); + if (value == null || value.length() == 0) { + return defaultValue; + } + byte b = Byte.parseByte(value); + getNumbers().put(key, b); + return b; + } + + public void setCompiler(String compiler) { + this.compiler = compiler; + AdaptiveCompiler.setDefaultCompiler(compiler); + } + + public String getCompiler() { + return compiler; + } + + @Override + public String toString() { + return JSONUtils.toJSONString(this); + } +} diff --git a/job-core/src/main/java/com/lts/job/core/compiler/Compiler.java b/job-core/src/main/java/com/lts/job/core/compiler/Compiler.java new file mode 100644 index 000000000..4ac5202a2 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/compiler/Compiler.java @@ -0,0 +1,21 @@ +package com.lts.job.core.compiler; + +import com.lts.job.core.extension.SPI; + +/** + * Compiler. (SPI, Singleton, ThreadSafe) + * @author william.liangf + */ +@SPI("javassist") +public interface Compiler { + + /** + * Compile java source code. + * + * @param code Java source code + * @param classLoader + * @return Compiled class + */ + Class compile(String code, ClassLoader classLoader); + +} diff --git a/job-core/src/main/java/com/lts/job/core/compiler/support/AbstractCompiler.java b/job-core/src/main/java/com/lts/job/core/compiler/support/AbstractCompiler.java new file mode 100755 index 000000000..4d48a803d --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/compiler/support/AbstractCompiler.java @@ -0,0 +1,69 @@ +/* + * Copyright 1999-2011 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.compiler.support; + +import com.lts.job.core.util.ClassHelper; +import com.lts.job.core.compiler.Compiler; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Abstract compiler. (SPI, Prototype, ThreadSafe) + * @author william.liangf + */ +public abstract class AbstractCompiler implements Compiler { + + private static final Pattern PACKAGE_PATTERN = Pattern.compile("package\\s+([$_a-zA-Z][$_a-zA-Z0-9\\.]*);"); + + private static final Pattern CLASS_PATTERN = Pattern.compile("class\\s+([$_a-zA-Z][$_a-zA-Z0-9]*)\\s+"); + + public Class compile(String code, ClassLoader classLoader) { + code = code.trim(); + Matcher matcher = PACKAGE_PATTERN.matcher(code); + String pkg; + if (matcher.find()) { + pkg = matcher.group(1); + } else { + pkg = ""; + } + matcher = CLASS_PATTERN.matcher(code); + String cls; + if (matcher.find()) { + cls = matcher.group(1); + } else { + throw new IllegalArgumentException("No such class name in " + code); + } + String className = pkg != null && pkg.length() > 0 ? pkg + "." + cls : cls; + try { + return Class.forName(className, true, ClassHelper.getCallerClassLoader(getClass())); + } catch (ClassNotFoundException e) { + if (!code.endsWith("}")) { + throw new IllegalStateException("The java code not endsWith \"}\", code: \n" + code + "\n"); + } + try { + return doCompile(className, code); + } catch (RuntimeException t) { + throw t; + } catch (Throwable t) { + throw new IllegalStateException("Failed to compile class, cause: " + t.getMessage() + ", class: " + className + ", code: \n" + code + "\n, stack: " + ClassUtils.toString(t)); + } + } + } + + protected abstract Class doCompile(String name, String source) throws Throwable; + +} diff --git a/job-core/src/main/java/com/lts/job/core/compiler/support/AdaptiveCompiler.java b/job-core/src/main/java/com/lts/job/core/compiler/support/AdaptiveCompiler.java new file mode 100755 index 000000000..d833ad59f --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/compiler/support/AdaptiveCompiler.java @@ -0,0 +1,48 @@ +/* + * Copyright 1999-2011 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.compiler.support; + + +import com.lts.job.core.extension.Adaptive; +import com.lts.job.core.extension.ExtensionLoader; +import com.lts.job.core.compiler.Compiler; + +/** + * AdaptiveCompiler. (SPI, Singleton, ThreadSafe) + * @author william.liangf + */ +@Adaptive +public class AdaptiveCompiler implements Compiler { + + private static volatile String DEFAULT_COMPILER; + + public static void setDefaultCompiler(String compiler) { + DEFAULT_COMPILER = compiler; + } + + public Class compile(String code, ClassLoader classLoader) { + Compiler compiler; + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(Compiler.class); + String name = DEFAULT_COMPILER; // copy reference + if (name != null && name.length() > 0) { + compiler = loader.getExtension(name); + } else { + compiler = loader.getDefaultExtension(); + } + return compiler.compile(code, classLoader); + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/compiler/support/ClassUtils.java b/job-core/src/main/java/com/lts/job/core/compiler/support/ClassUtils.java new file mode 100755 index 000000000..21a893135 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/compiler/support/ClassUtils.java @@ -0,0 +1,413 @@ +/* + * Copyright 1999-2011 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.compiler.support; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * ClassUtils. (Tool, Static, ThreadSafe) + * + * @author william.liangf + */ +public class ClassUtils { + + public static final String CLASS_EXTENSION = ".class"; + + public static final String JAVA_EXTENSION = ".java"; + + public static Object newInstance(String name) { + try { + return forName(name).newInstance(); + } catch (InstantiationException e) { + throw new IllegalStateException(e.getMessage(), e); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static Class forName(String[] packages, String className) { + try { + return _forName(className); + } catch (ClassNotFoundException e) { + if (packages != null && packages.length > 0) { + for (String pkg : packages) { + try { + return _forName(pkg + "." + className); + } catch (ClassNotFoundException e2) { + } + } + } + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static Class forName(String className) { + try { + return _forName(className); + } catch (ClassNotFoundException e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static Class _forName(String className) throws ClassNotFoundException { + if ("boolean".equals(className)) + return boolean.class; + if ("byte".equals(className)) + return byte.class; + if ("char".equals(className)) + return char.class; + if ("short".equals(className)) + return short.class; + if ("int".equals(className)) + return int.class; + if ("long".equals(className)) + return long.class; + if ("float".equals(className)) + return float.class; + if ("double".equals(className)) + return double.class; + if ("boolean[]".equals(className)) + return boolean[].class; + if ("byte[]".equals(className)) + return byte[].class; + if ("char[]".equals(className)) + return char[].class; + if ("short[]".equals(className)) + return short[].class; + if ("int[]".equals(className)) + return int[].class; + if ("long[]".equals(className)) + return long[].class; + if ("float[]".equals(className)) + return float[].class; + if ("double[]".equals(className)) + return double[].class; + try { + return arrayForName(className); + } catch (ClassNotFoundException e) { + if (className.indexOf('.') == -1) { // 尝试java.lang包 + try { + return arrayForName("java.lang." + className); + } catch (ClassNotFoundException e2) { + // 忽略尝试异常, 抛出原始异常 + } + } + throw e; + } + } + + private static Class arrayForName(String className) throws ClassNotFoundException { + return Class.forName(className.endsWith("[]") + ? "[L" + className.substring(0, className.length() - 2) + ";" + : className, true, Thread.currentThread().getContextClassLoader()); + } + + public static Class getBoxedClass(Class type) { + if (type == boolean.class) { + return Boolean.class; + } else if (type == char.class) { + return Character.class; + } else if (type == byte.class) { + return Byte.class; + } else if (type == short.class) { + return Short.class; + } else if (type == int.class) { + return Integer.class; + } else if (type == long.class) { + return Long.class; + } else if (type == float.class) { + return Float.class; + } else if (type == double.class) { + return Double.class; + } else { + return type; + } + } + + public static Boolean boxed(boolean v) { + return Boolean.valueOf(v); + } + + public static Character boxed(char v) { + return Character.valueOf(v); + } + + public static Byte boxed(byte v) { + return Byte.valueOf(v); + } + + public static Short boxed(short v) { + return Short.valueOf(v); + } + + public static Integer boxed(int v) { + return Integer.valueOf(v); + } + + public static Long boxed(long v) { + return Long.valueOf(v); + } + + public static Float boxed(float v) { + return Float.valueOf(v); + } + + public static Double boxed(double v) { + return Double.valueOf(v); + } + + public static Object boxed(Object v) { + return v; + } + + public static boolean unboxed(Boolean v) { + return v == null ? false : v.booleanValue(); + } + + public static char unboxed(Character v) { + return v == null ? '\0' : v.charValue(); + } + + public static byte unboxed(Byte v) { + return v == null ? 0 : v.byteValue(); + } + + public static short unboxed(Short v) { + return v == null ? 0 : v.shortValue(); + } + + public static int unboxed(Integer v) { + return v == null ? 0 : v.intValue(); + } + + public static long unboxed(Long v) { + return v == null ? 0 : v.longValue(); + } + + public static float unboxed(Float v) { + return v == null ? 0 : v.floatValue(); + } + + public static double unboxed(Double v) { + return v == null ? 0 : v.doubleValue(); + } + + public static Object unboxed(Object v) { + return v; + } + + public static boolean isNotEmpty(Object object) { + return getSize(object) > 0; + } + + public static int getSize(Object object) { + if (object == null) { + return 0; + } if (object instanceof Collection) { + return ((Collection)object).size(); + } else if (object instanceof Map) { + return ((Map)object).size(); + } else if (object.getClass().isArray()) { + return Array.getLength(object); + } else { + return -1; + } + } + + public static URI toURI(String name) { + try { + return new URI(name); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public static Class getGenericClass(Class cls) { + return getGenericClass(cls, 0); + } + + public static Class getGenericClass(Class cls, int i) { + try { + ParameterizedType parameterizedType = ((ParameterizedType) cls.getGenericInterfaces()[0]); + Object genericClass = parameterizedType.getActualTypeArguments()[i]; + if (genericClass instanceof ParameterizedType) { // 处理多级泛型 + return (Class) ((ParameterizedType) genericClass).getRawType(); + } else if (genericClass instanceof GenericArrayType) { // 处理数组泛型 + return (Class) ((GenericArrayType) genericClass).getGenericComponentType(); + } else if (genericClass != null) { + return (Class) genericClass; + } + } catch (Throwable e) { + } + if (cls.getSuperclass() != null) { + return getGenericClass(cls.getSuperclass(), i); + } else { + throw new IllegalArgumentException(cls.getName() + " generic type undefined!"); + } + } + + public static boolean isBeforeJava5(String javaVersion) { + return (javaVersion == null || javaVersion.length() == 0 || "1.0".equals(javaVersion) + || "1.1".equals(javaVersion) || "1.2".equals(javaVersion) + || "1.3".equals(javaVersion) || "1.4".equals(javaVersion)); + } + + public static boolean isBeforeJava6(String javaVersion) { + return isBeforeJava5(javaVersion) || "1.5".equals(javaVersion); + } + + public static String toString(Throwable e) { + StringWriter w = new StringWriter(); + PrintWriter p = new PrintWriter(w); + p.print(e.getClass().getName() + ": "); + if (e.getMessage() != null) { + p.print(e.getMessage() + "\n"); + } + p.println(); + try { + e.printStackTrace(p); + return w.toString(); + } finally { + p.close(); + } + } + + private static final int JIT_LIMIT = 5 * 1024; + + public static void checkBytecode(String name, byte[] bytecode) { + if (bytecode.length > JIT_LIMIT) { + System.err.println("The template bytecode too long, may be affect the JIT compiler. template class: " + name); + } + } + + public static String getSizeMethod(Class cls) { + try { + return cls.getMethod("size", new Class[0]).getName() + "()"; + } catch (NoSuchMethodException e) { + try { + return cls.getMethod("length", new Class[0]).getName() + "()"; + } catch (NoSuchMethodException e2) { + try { + return cls.getMethod("getSize", new Class[0]).getName() + "()"; + } catch (NoSuchMethodException e3) { + try { + return cls.getMethod("getLength", new Class[0]).getName() + "()"; + } catch (NoSuchMethodException e4) { + return null; + } + } + } + } + } + + public static String getMethodName(Method method, Class[] parameterClasses, String rightCode) { + if (method.getParameterTypes().length > parameterClasses.length) { + Class[] types = method.getParameterTypes(); + StringBuilder buf = new StringBuilder(rightCode); + for (int i = parameterClasses.length; i < types.length; i ++) { + if (buf.length() > 0) { + buf.append(","); + } + Class type = types[i]; + String def; + if (type == boolean.class) { + def = "false"; + } else if (type == char.class) { + def = "\'\\0\'"; + } else if (type == byte.class + || type == short.class + || type == int.class + || type == long.class + || type == float.class + || type == double.class) { + def = "0"; + } else { + def = "null"; + } + buf.append(def); + } + } + return method.getName() + "(" + rightCode + ")"; + } + + public static Method searchMethod(Class currentClass, String name, Class[] parameterTypes) throws NoSuchMethodException { + if (currentClass == null) { + throw new NoSuchMethodException("class == null"); + } + try { + return currentClass.getMethod(name, parameterTypes); + } catch (NoSuchMethodException e) { + for (Method method : currentClass.getMethods()) { + if (method.getName().equals(name) + && parameterTypes.length == method.getParameterTypes().length + && Modifier.isPublic(method.getModifiers())) { + if (parameterTypes.length > 0) { + Class[] types = method.getParameterTypes(); + boolean match = true; + for (int i = 0; i < parameterTypes.length; i ++) { + if (! types[i].isAssignableFrom(parameterTypes[i])) { + match = false; + break; + } + } + if (! match) { + continue; + } + } + return method; + } + } + throw e; + } + } + + public static String getInitCode(Class type) { + if (byte.class.equals(type) + || short.class.equals(type) + || int.class.equals(type) + || long.class.equals(type) + || float.class.equals(type) + || double.class.equals(type)) { + return "0"; + } else if (char.class.equals(type)) { + return "'\\0'"; + } else if (boolean.class.equals(type)) { + return "false"; + } else { + return "null"; + } + } + + public static Map toMap(Map.Entry[] entries) { + Map map = new HashMap(); + if (entries != null && entries.length > 0) { + for (Map.Entry enrty : entries) { + map.put(enrty.getKey(), enrty.getValue()); + } + } + return map; + } + + private ClassUtils() {} + +} diff --git a/job-core/src/main/java/com/lts/job/core/compiler/support/JavassistCompiler.java b/job-core/src/main/java/com/lts/job/core/compiler/support/JavassistCompiler.java new file mode 100755 index 000000000..f73fe6128 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/compiler/support/JavassistCompiler.java @@ -0,0 +1,119 @@ +/* + * Copyright 1999-2011 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.compiler.support; + +import com.lts.job.core.util.ClassHelper; +import javassist.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * JavassistCompiler. (SPI, Singleton, ThreadSafe) + * @author william.liangf + */ +public class JavassistCompiler extends AbstractCompiler { + + private static final Pattern IMPORT_PATTERN = Pattern.compile("import\\s+([\\w\\.\\*]+);\n"); + + private static final Pattern EXTENDS_PATTERN = Pattern.compile("\\s+extends\\s+([\\w\\.]+)[^\\{]*\\{\n"); + + private static final Pattern IMPLEMENTS_PATTERN = Pattern.compile("\\s+implements\\s+([\\w\\.]+)\\s*\\{\n"); + + private static final Pattern METHODS_PATTERN = Pattern.compile("\n(private|public|protected)\\s+"); + + private static final Pattern FIELD_PATTERN = Pattern.compile("[^\n]+=[^\n]+;"); + + @Override + public Class doCompile(String name, String source) throws Throwable { + int i = name.lastIndexOf('.'); + String className = i < 0 ? name : name.substring(i + 1); + ClassPool pool = new ClassPool(true); + pool.appendClassPath(new LoaderClassPath(ClassHelper.getCallerClassLoader(getClass()))); + Matcher matcher = IMPORT_PATTERN.matcher(source); + List importPackages = new ArrayList(); + Map fullNames = new HashMap(); + while (matcher.find()) { + String pkg = matcher.group(1); + if (pkg.endsWith(".*")) { + String pkgName = pkg.substring(0, pkg.length() - 2); + pool.importPackage(pkgName); + importPackages.add(pkgName); + } else { + int pi = pkg.lastIndexOf('.'); + if (pi > 0) { + String pkgName = pkg.substring(0, pi); + pool.importPackage(pkgName); + importPackages.add(pkgName); + fullNames.put(pkg.substring(pi + 1), pkg); + } + } + } + String[] packages = importPackages.toArray(new String[0]); + matcher = EXTENDS_PATTERN.matcher(source); + CtClass cls; + if (matcher.find()) { + String extend = matcher.group(1).trim(); + String extendClass; + if (extend.contains(".")) { + extendClass = extend; + } else if (fullNames.containsKey(extend)) { + extendClass = fullNames.get(extend); + } else { + extendClass = ClassUtils.forName(packages, extend).getName(); + } + cls = pool.makeClass(name, pool.get(extendClass)); + } else { + cls = pool.makeClass(name); + } + matcher = IMPLEMENTS_PATTERN.matcher(source); + if (matcher.find()) { + String[] ifaces = matcher.group(1).trim().split("\\,"); + for (String iface : ifaces) { + iface = iface.trim(); + String ifaceClass; + if (iface.contains(".")) { + ifaceClass = iface; + } else if (fullNames.containsKey(iface)) { + ifaceClass = fullNames.get(iface); + } else { + ifaceClass = ClassUtils.forName(packages, iface).getName(); + } + cls.addInterface(pool.get(ifaceClass)); + } + } + String body = source.substring(source.indexOf("{") + 1, source.length() - 1); + String[] methods = METHODS_PATTERN.split(body); + for (String method : methods) { + method = method.trim(); + if (method.length() > 0) { + if (method.startsWith(className)) { + cls.addConstructor(CtNewConstructor.make("public " + method, cls)); + } else if (FIELD_PATTERN.matcher(method).matches()) { + cls.addField(CtField.make("private " + method, cls)); + } else { + cls.addMethod(CtNewMethod.make("public " + method, cls)); + } + } + } + return cls.toClass(ClassHelper.getCallerClassLoader(getClass()), JavassistCompiler.class.getProtectionDomain()); + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/compiler/support/JdkCompiler.java b/job-core/src/main/java/com/lts/job/core/compiler/support/JdkCompiler.java new file mode 100755 index 000000000..82586af6d --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/compiler/support/JdkCompiler.java @@ -0,0 +1,268 @@ +/* + * Copyright 1999-2011 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.compiler.support; + +import com.lts.job.core.util.ClassHelper; + +import javax.tools.*; +import javax.tools.JavaFileObject.Kind; +import java.io.*; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.*; + +/** + * JdkCompiler. (SPI, Singleton, ThreadSafe) + * + * @author william.liangf + */ +public class JdkCompiler extends AbstractCompiler { + + private final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + + private final DiagnosticCollector diagnosticCollector = new DiagnosticCollector(); + + private final ClassLoaderImpl classLoader; + + private final JavaFileManagerImpl javaFileManager; + + private volatile List options; + + public JdkCompiler(){ + options = new ArrayList(); + options.add("-target"); + options.add("1.7"); + StandardJavaFileManager manager = compiler.getStandardFileManager(diagnosticCollector, null, null); + final ClassLoader loader = Thread.currentThread().getContextClassLoader(); + if (loader instanceof URLClassLoader + && (! loader.getClass().getName().equals("sun.misc.Launcher$AppClassLoader"))) { + try { + URLClassLoader urlClassLoader = (URLClassLoader) loader; + List files = new ArrayList(); + for (URL url : urlClassLoader.getURLs()) { + files.add(new File(url.getFile())); + } + manager.setLocation(StandardLocation.CLASS_PATH, files); + } catch (IOException e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + classLoader = AccessController.doPrivileged(new PrivilegedAction() { + public ClassLoaderImpl run() { + return new ClassLoaderImpl(loader); + } + }); + javaFileManager = new JavaFileManagerImpl(manager, classLoader); + } + + @Override + public Class doCompile(String name, String sourceCode) throws Throwable { + int i = name.lastIndexOf('.'); + String packageName = i < 0 ? "" : name.substring(0, i); + String className = i < 0 ? name : name.substring(i + 1); + JavaFileObjectImpl javaFileObject = new JavaFileObjectImpl(className, sourceCode); + javaFileManager.putFileForInput(StandardLocation.SOURCE_PATH, packageName, + className + ClassUtils.JAVA_EXTENSION, javaFileObject); + Boolean result = compiler.getTask(null, javaFileManager, diagnosticCollector, options, + null, Arrays.asList(new JavaFileObject[]{javaFileObject})).call(); + if (result == null || ! result.booleanValue()) { + throw new IllegalStateException("Compilation failed. class: " + name + ", diagnostics: " + diagnosticCollector); + } + return classLoader.loadClass(name); + } + + private final class ClassLoaderImpl extends ClassLoader { + + private final Map classes = new HashMap(); + + ClassLoaderImpl(final ClassLoader parentClassLoader) { + super(parentClassLoader); + } + + Collection files() { + return Collections.unmodifiableCollection(classes.values()); + } + + @Override + protected Class findClass(final String qualifiedClassName) throws ClassNotFoundException { + JavaFileObject file = classes.get(qualifiedClassName); + if (file != null) { + byte[] bytes = ((JavaFileObjectImpl) file).getByteCode(); + return defineClass(qualifiedClassName, bytes, 0, bytes.length); + } + try { + return ClassHelper.forNameWithCallerClassLoader(qualifiedClassName, getClass()); + } catch (ClassNotFoundException nf) { + return super.findClass(qualifiedClassName); + } + } + + void add(final String qualifiedClassName, final JavaFileObject javaFile) { + classes.put(qualifiedClassName, javaFile); + } + + @Override + protected synchronized Class loadClass(final String name, final boolean resolve) throws ClassNotFoundException { + return super.loadClass(name, resolve); + } + + @Override + public InputStream getResourceAsStream(final String name) { + if (name.endsWith(ClassUtils.CLASS_EXTENSION)) { + String qualifiedClassName = name.substring(0, name.length() - ClassUtils.CLASS_EXTENSION.length()).replace('/', '.'); + JavaFileObjectImpl file = (JavaFileObjectImpl) classes.get(qualifiedClassName); + if (file != null) { + return new ByteArrayInputStream(file.getByteCode()); + } + } + return super.getResourceAsStream(name); + } + } + + private static final class JavaFileObjectImpl extends SimpleJavaFileObject { + + private ByteArrayOutputStream bytecode; + + private final CharSequence source; + + public JavaFileObjectImpl(final String baseName, final CharSequence source){ + super(ClassUtils.toURI(baseName + ClassUtils.JAVA_EXTENSION), Kind.SOURCE); + this.source = source; + } + + JavaFileObjectImpl(final String name, final Kind kind){ + super(ClassUtils.toURI(name), kind); + source = null; + } + + public JavaFileObjectImpl(URI uri, Kind kind){ + super(uri, kind); + source = null; + } + + @Override + public CharSequence getCharContent(final boolean ignoreEncodingErrors) throws UnsupportedOperationException { + if (source == null) { + throw new UnsupportedOperationException("source == null"); + } + return source; + } + + @Override + public InputStream openInputStream() { + return new ByteArrayInputStream(getByteCode()); + } + + @Override + public OutputStream openOutputStream() { + return bytecode = new ByteArrayOutputStream(); + } + + public byte[] getByteCode() { + return bytecode.toByteArray(); + } + } + + private static final class JavaFileManagerImpl extends ForwardingJavaFileManager { + + private final ClassLoaderImpl classLoader; + + private final Map fileObjects = new HashMap(); + + public JavaFileManagerImpl(JavaFileManager fileManager, ClassLoaderImpl classLoader) { + super(fileManager); + this.classLoader = classLoader; + } + + @Override + public FileObject getFileForInput(Location location, String packageName, String relativeName) throws IOException { + FileObject o = fileObjects.get(uri(location, packageName, relativeName)); + if (o != null) + return o; + return super.getFileForInput(location, packageName, relativeName); + } + + public void putFileForInput(StandardLocation location, String packageName, String relativeName, JavaFileObject file) { + fileObjects.put(uri(location, packageName, relativeName), file); + } + + private URI uri(Location location, String packageName, String relativeName) { + return ClassUtils.toURI(location.getName() + '/' + packageName + '/' + relativeName); + } + + @Override + public JavaFileObject getJavaFileForOutput(Location location, String qualifiedName, Kind kind, FileObject outputFile) + throws IOException { + JavaFileObject file = new JavaFileObjectImpl(qualifiedName, kind); + classLoader.add(qualifiedName, file); + return file; + } + + @Override + public ClassLoader getClassLoader(Location location) { + return classLoader; + } + + @Override + public String inferBinaryName(Location loc, JavaFileObject file) { + if (file instanceof JavaFileObjectImpl) + return file.getName(); + return super.inferBinaryName(loc, file); + } + + @Override + public Iterable list(Location location, String packageName, Set kinds, boolean recurse) + throws IOException { + Iterable result = super.list(location, packageName, kinds, recurse); + + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + List urlList = new ArrayList(); + Enumeration e = contextClassLoader.getResources("com"); + while (e.hasMoreElements()) { + urlList.add(e.nextElement()); + } + + ArrayList files = new ArrayList(); + + if (location == StandardLocation.CLASS_PATH && kinds.contains(Kind.CLASS)) { + for (JavaFileObject file : fileObjects.values()) { + if (file.getKind() == Kind.CLASS && file.getName().startsWith(packageName)) { + files.add(file); + } + } + + files.addAll(classLoader.files()); + } else if (location == StandardLocation.SOURCE_PATH && kinds.contains(Kind.SOURCE)) { + for (JavaFileObject file : fileObjects.values()) { + if (file.getKind() == Kind.SOURCE && file.getName().startsWith(packageName)) { + files.add(file); + } + } + } + + for (JavaFileObject file : result) { + files.add(file); + } + + return files; + } + } + + +} diff --git a/job-core/src/main/java/com/lts/job/core/constant/Constants.java b/job-core/src/main/java/com/lts/job/core/constant/Constants.java index 67fa6d104..b50f66cf8 100644 --- a/job-core/src/main/java/com/lts/job/core/constant/Constants.java +++ b/job-core/src/main/java/com/lts/job/core/constant/Constants.java @@ -1,6 +1,8 @@ package com.lts.job.core.constant; +import java.util.regex.Pattern; + /** * @author Robert HG (254963746@qq.com) on 7/24/14. * 一些配置常量 @@ -48,4 +50,5 @@ public interface Constants { */ public static final int DEFAULT_REGISTRY_RETRY_PERIOD = 5 * 1000; + public static final Pattern COMMA_SPLIT_PATTERN = Pattern.compile("\\s*[,]+\\s*"); } diff --git a/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java b/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java deleted file mode 100644 index bb3b4700b..000000000 --- a/job-core/src/main/java/com/lts/job/core/domain/JobNodeConfig.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.lts.job.core.domain; - -import com.lts.job.core.cluster.NodeType; -import com.lts.job.core.util.JSONUtils; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author Robert HG (254963746@qq.com) on 8/20/14. - * 任务节点配置 - */ -public class JobNodeConfig { - - // 节点是否可用 - private boolean available = true; - // 应用节点组 - private String nodeGroup; - // 唯一标识 - private String identity; - // 工作线程 - private int workThreads; - // 节点类型 - private NodeType nodeType; - // 注册中心 地址 - private String registryAddress; - // 远程连接超时时间 - private int invokeTimeoutMillis; - // 监听端口 - private int listenPort; - // 任务信息存储路径(譬如TaskTracker反馈任务信息给JobTracker, JobTracker down掉了, 那么存储下来等待JobTracker可用时再发送) - private String jobInfoSavePath; - // 集群名字 - private String clusterName; - - private final Map parameters = new HashMap(); - - public String getClusterName() { - return clusterName; - } - - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - - public String getNodeGroup() { - return nodeGroup; - } - - public void setNodeGroup(String nodeGroup) { - this.nodeGroup = nodeGroup; - } - - public String getIdentity() { - return identity; - } - - public void setIdentity(String identity) { - this.identity = identity; - } - - public int getWorkThreads() { - return workThreads; - } - - public void setWorkThreads(int workThreads) { - this.workThreads = workThreads; - } - - public NodeType getNodeType() { - return nodeType; - } - - public void setNodeType(NodeType nodeType) { - this.nodeType = nodeType; - } - - public String getRegistryAddress() { - return registryAddress; - } - - public void setRegistryAddress(String registryAddress) { - this.registryAddress = registryAddress; - } - - public int getInvokeTimeoutMillis() { - return invokeTimeoutMillis; - } - - public void setInvokeTimeoutMillis(int invokeTimeoutMillis) { - this.invokeTimeoutMillis = invokeTimeoutMillis; - } - - public int getListenPort() { - return listenPort; - } - - public void setListenPort(int listenPort) { - this.listenPort = listenPort; - } - - public String getJobInfoSavePath() { - return jobInfoSavePath; - } - - public void setJobInfoSavePath(String jobInfoSavePath) { - this.jobInfoSavePath = jobInfoSavePath + "/.lts"; - } - - public String getFilePath() { - return jobInfoSavePath + "/" + nodeType + "/" + nodeGroup + "/"; - } - - public boolean isAvailable() { - return available; - } - - public void setAvailable(boolean available) { - this.available = available; - } - - public void setParameter(String key, String value) { - parameters.put(key, value); - } - - public T getParameter(String key) { - return (T)parameters.get(key); - } - - public T getParameter(String key, T defaultValue) { - Object value = parameters.get(key); - if (value == null) { - return defaultValue; - } - return (T)value; - } - - @Override - public String toString() { - return JSONUtils.toJSONString(this); - } -} diff --git a/job-core/src/main/java/com/lts/job/core/extension/Activate.java b/job-core/src/main/java/com/lts/job/core/extension/Activate.java new file mode 100644 index 000000000..0c94855df --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/Activate.java @@ -0,0 +1,78 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension; + +import java.lang.annotation.*; + +/** + * Activate + *

+ * 对于可以被框架中自动激活加载扩展，此Annotation用于配置扩展被自动激活加载条件。 + * 比如，过滤扩展，有多个实现，使用Activate Annotation的扩展可以根据条件被自动加载。 + *

    + *
  1. {@link Activate#group()}生效的Group。具体的有哪些Group值由框架SPI给出。 + *
  2. {@link Activate#value()}在{@link com.lts.job.core.cluster.Config}中Key集合中有，则生效。 + *
+ * + *

+ * 底层框架SPI提供者通过{@link ExtensionLoader}的{@link ExtensionLoader#getAdaptiveExtension()}方法 + * 获得条件的扩展。 + * + * @author william.liangf + * @author ding.lid + * @export + * @see SPI + * @see ExtensionLoader + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface Activate { + /** + * Group过滤条件。 + *
+ * 包含{@link ExtensionLoader#getAdaptiveExtension()}的group参数给的值，则返回扩展。 + *
+ * 如没有Group设置，则不过滤。 + */ + String[] group() default {}; + + /** + * Key过滤条件。包含{@link ExtensionLoader#getAdaptiveExtension()}的URL的参数Key中有，则返回扩展。 + *

+ * 示例：
+ * 注解的值 @Activate("cache,validatioin")， + * 则{@link ExtensionLoader#getAdaptiveExtension()}的URL的参数有cacheKey，或是validatioin则返回扩展。 + *
+ * 如没有设置，则不过滤。 + */ + String[] value() default {}; + + /** + * 排序信息，可以不提供。 + */ + String[] before() default {}; + + /** + * 排序信息，可以不提供。 + */ + String[] after() default {}; + + /** + * 排序信息，可以不提供。 + */ + int order() default 0; +} \ No newline at end of file diff --git a/job-core/src/main/java/com/lts/job/core/extension/Adaptive.java b/job-core/src/main/java/com/lts/job/core/extension/Adaptive.java new file mode 100644 index 000000000..c2478b989 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/Adaptive.java @@ -0,0 +1,53 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension; + +import com.lts.job.core.cluster.Config; + +import java.lang.annotation.*; + +/** + * 在{@link ExtensionLoader}生成Extension的Adaptive Instance时，为{@link ExtensionLoader}提供信息。 + * @export + * + * @see ExtensionLoader + * @see com.lts.job.core.cluster.Config + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface Adaptive { + + /** + * 从{@link Config}的Key名，对应的Value作为要Adapt成的Extension名。 + *

+ * 如果{@link Config}这些Key都没有Value，使用 用 缺省的扩展（在接口的{@link SPI}中设定的值）。
+ * 比如，String[] {"key1", "key2"}，表示 + *

    + *
  1. 先在URL上找key1的Value作为要Adapt成的Extension名； + *
  2. key1没有Value，则使用key2的Value作为要Adapt成的Extension名。 + *
  3. key2没有Value，使用缺省的扩展。 + *
  4. 如果没有设定缺省扩展，则方法调用会抛出{@link IllegalStateException}。 + *
+ *

+ * 如果不设置则缺省使用Extension接口类名的点分隔小写字串。
+ * 即对于Extension接口{@code com.lts.job.core.XxxYyyService}的缺省值为String[] {"xxx.yyy.service"} + * + * @see SPI#value() + */ + String[] value() default {}; + +} \ No newline at end of file diff --git a/job-core/src/main/java/com/lts/job/core/extension/ExtensionFactory.java b/job-core/src/main/java/com/lts/job/core/extension/ExtensionFactory.java new file mode 100644 index 000000000..0a4529632 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/ExtensionFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension; + +/** + * Created by hugui on 5/18/15. + */ +@SPI +public interface ExtensionFactory { + + /** + * Get extension. + * + * @param type object type. + * @param name object name. + * @return object instance. + */ + T getExtension(Class type, String name); + +} + diff --git a/job-core/src/main/java/com/lts/job/core/extension/ExtensionLoader.java b/job-core/src/main/java/com/lts/job/core/extension/ExtensionLoader.java new file mode 100644 index 000000000..721bb7466 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/ExtensionLoader.java @@ -0,0 +1,735 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension; + +import com.lts.job.core.cluster.Config; +import com.lts.job.core.util.ConcurrentHashSet; +import com.lts.job.core.util.Holder; +import com.lts.job.core.util.StringUtils; +import com.lts.job.core.compiler.Compiler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.regex.Pattern; + +/** + * Dubbo使用的扩展点获取。

+ *

    + *
  • 自动注入关联扩展点。
    • + *
  • 自动Wrap上扩展点的Wrap类。
    • + *
  • 缺省获得的的扩展点是一个Adaptive Instance。 + *
+ * + * @author william.liangf + * @author ding.lid + * @author Robert HG (254963746@qq.com) + * @see JDK5.0的自动发现机制实现 + * @see SPI + * @see Adaptive + * @see Activate + */ +public class ExtensionLoader { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExtensionLoader.class); + + private static final String DUBBO_DIRECTORY = "META-INF/lts/"; + + private static final String DUBBO_INTERNAL_DIRECTORY = DUBBO_DIRECTORY + "internal/"; + + private static final Pattern NAME_SEPARATOR = Pattern.compile("\\s*[,]+\\s*"); + + private static final ConcurrentMap, ExtensionLoader> EXTENSION_LOADERS = new ConcurrentHashMap, ExtensionLoader>(); + + private static final ConcurrentMap, Object> EXTENSION_INSTANCES = new ConcurrentHashMap, Object>(); + + private final Class type; + + private final ExtensionFactory objectFactory; + + private final ConcurrentMap, String> cachedNames = new ConcurrentHashMap, String>(); + + private final Holder>> cachedClasses = new Holder>>(); + + private final Map cachedActivates = new ConcurrentHashMap(); + + private volatile Class cachedAdaptiveClass = null; + + private final ConcurrentMap> cachedInstances = new ConcurrentHashMap>(); + + // 配置在@SPI + private String cachedDefaultName; + + private final Holder cachedAdaptiveInstance = new Holder(); + private volatile Throwable createAdaptiveInstanceError; + + private Set> cachedWrapperClasses; + + private Map exceptions = new ConcurrentHashMap(); + + private static boolean withExtensionAnnotation(Class type) { + return type.isAnnotationPresent(SPI.class); + } + + @SuppressWarnings("unchecked") + public static ExtensionLoader getExtensionLoader(Class type) { + if (type == null) + throw new IllegalArgumentException("Extension type == null"); + if (!type.isInterface()) { + throw new IllegalArgumentException("Extension type(" + type + ") is not interface!"); + } + if (!withExtensionAnnotation(type)) { + throw new IllegalArgumentException("Extension type(" + type + + ") is not extension, because WITHOUT @" + SPI.class.getSimpleName() + " Annotation!"); + } + + ExtensionLoader loader = (ExtensionLoader) EXTENSION_LOADERS.get(type); + if (loader == null) { + EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader(type)); + loader = (ExtensionLoader) EXTENSION_LOADERS.get(type); + } + return loader; + } + + private ExtensionLoader(Class type) { + this.type = type; + objectFactory = (type == ExtensionFactory.class ? null : ExtensionLoader.getExtensionLoader(ExtensionFactory.class).getAdaptiveExtension()); + } + + public String getExtensionName(T extensionInstance) { + return getExtensionName(extensionInstance.getClass()); + } + + public String getExtensionName(Class extensionClass) { + return cachedNames.get(extensionClass); + } + + private boolean isMatchGroup(String group, String[] groups) { + if (group == null || group.length() == 0) { + return true; + } + if (groups != null && groups.length > 0) { + for (String g : groups) { + if (group.equals(g)) { + return true; + } + } + } + return false; + } + + /** + * 返回扩展点实例，如果没有指定的扩展点或是还没加载（即实例化）则返回null。注意：此方法不会触发扩展点的加载。 + *

+ * 一般应该调用{@link #getExtension(String)}方法获得扩展，这个方法会触发扩展点加载。 + * + * @see #getExtension(String) + */ + @SuppressWarnings("unchecked") + public T getLoadedExtension(String name) { + if (name == null || name.length() == 0) + throw new IllegalArgumentException("Extension name == null"); + Holder holder = cachedInstances.get(name); + if (holder == null) { + cachedInstances.putIfAbsent(name, new Holder()); + holder = cachedInstances.get(name); + } + return (T) holder.get(); + } + + /** + * 返回已经加载的扩展点的名字。 + *

+ * 一般应该调用{@link #getSupportedExtensions()}方法获得扩展，这个方法会返回所有的扩展点。 + * + * @see #getSupportedExtensions() + */ + public Set getLoadedExtensions() { + return Collections.unmodifiableSet(new TreeSet(cachedInstances.keySet())); + } + + /** + * 返回指定名字的扩展。如果指定名字的扩展不存在，则抛异常 {@link IllegalStateException}. + * + * @param name + * @return + */ + @SuppressWarnings("unchecked") + public T getExtension(String name) { + if (name == null || name.length() == 0) + throw new IllegalArgumentException("Extension name == null"); + if ("true".equals(name)) { + return getDefaultExtension(); + } + Holder holder = cachedInstances.get(name); + if (holder == null) { + cachedInstances.putIfAbsent(name, new Holder()); + holder = cachedInstances.get(name); + } + Object instance = holder.get(); + if (instance == null) { + synchronized (holder) { + instance = holder.get(); + if (instance == null) { + instance = createExtension(name); + holder.set(instance); + } + } + } + return (T) instance; + } + + /** + * 返回缺省的扩展，如果没有设置则返回null。 + */ + public T getDefaultExtension() { + getExtensionClasses(); + if (null == cachedDefaultName || cachedDefaultName.length() == 0 + || "true".equals(cachedDefaultName)) { + return null; + } + return getExtension(cachedDefaultName); + } + + public boolean hasExtension(String name) { + if (name == null || name.length() == 0) + throw new IllegalArgumentException("Extension name == null"); + try { + return getExtensionClass(name) != null; + } catch (Throwable t) { + return false; + } + } + + public Set getSupportedExtensions() { + Map> clazzes = getExtensionClasses(); + return Collections.unmodifiableSet(new TreeSet(clazzes.keySet())); + } + + /** + * 返回缺省的扩展点名，如果没有设置缺省则返回null。 + */ + public String getDefaultExtensionName() { + getExtensionClasses(); + return cachedDefaultName; + } + + /** + * 编程方式添加新扩展点。 + * + * @param name 扩展点名 + * @param clazz 扩展点类 + * @throws IllegalStateException 要添加扩展点名已经存在。 + */ + public void addExtension(String name, Class clazz) { + getExtensionClasses(); // load classes + + if (!type.isAssignableFrom(clazz)) { + throw new IllegalStateException("Input type " + + clazz + "not implement Extension " + type); + } + if (clazz.isInterface()) { + throw new IllegalStateException("Input type " + + clazz + "can not be interface!"); + } + + if (!clazz.isAnnotationPresent(Adaptive.class)) { + if (StringUtils.isEmpty(name)) { + throw new IllegalStateException("Extension name is blank (Extension " + type + ")!"); + } + if (cachedClasses.get().containsKey(name)) { + throw new IllegalStateException("Extension name " + + name + " already existed(Extension " + type + ")!"); + } + + cachedNames.put(clazz, name); + cachedClasses.get().put(name, clazz); + } else { + if (cachedAdaptiveClass != null) { + throw new IllegalStateException("Adaptive Extension already existed(Extension " + type + ")!"); + } + + cachedAdaptiveClass = clazz; + } + } + + @SuppressWarnings("unchecked") + public T getAdaptiveExtension() { + Object instance = cachedAdaptiveInstance.get(); + if (instance == null) { + if (createAdaptiveInstanceError == null) { + synchronized (cachedAdaptiveInstance) { + instance = cachedAdaptiveInstance.get(); + if (instance == null) { + try { + instance = createAdaptiveExtension(); + cachedAdaptiveInstance.set(instance); + } catch (Throwable t) { + createAdaptiveInstanceError = t; + throw new IllegalStateException("fail to create adaptive instance: " + t.toString(), t); + } + } + } + } else { + throw new IllegalStateException("fail to create adaptive instance: " + createAdaptiveInstanceError.toString(), createAdaptiveInstanceError); + } + } + + return (T) instance; + } + + private IllegalStateException findException(String name) { + for (Map.Entry entry : exceptions.entrySet()) { + if (entry.getKey().toLowerCase().contains(name.toLowerCase())) { + return entry.getValue(); + } + } + StringBuilder buf = new StringBuilder("No such extension " + type.getName() + " by name " + name); + + + int i = 1; + for (Map.Entry entry : exceptions.entrySet()) { + if (i == 1) { + buf.append(", possible causes: "); + } + + buf.append("\r\n("); + buf.append(i++); + buf.append(") "); + buf.append(entry.getKey()); + buf.append(":\r\n"); + buf.append(StringUtils.toString(entry.getValue())); + } + return new IllegalStateException(buf.toString()); + } + + @SuppressWarnings("unchecked") + private T createExtension(String name) { + Class clazz = getExtensionClasses().get(name); + if (clazz == null) { + throw findException(name); + } + try { + T instance = (T) EXTENSION_INSTANCES.get(clazz); + if (instance == null) { + EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance()); + instance = (T) EXTENSION_INSTANCES.get(clazz); + } + injectExtension(instance); + Set> wrapperClasses = cachedWrapperClasses; + if (wrapperClasses != null && wrapperClasses.size() > 0) { + for (Class wrapperClass : wrapperClasses) { + instance = injectExtension((T) wrapperClass.getConstructor(type).newInstance(instance)); + } + } + return instance; + } catch (Throwable t) { + throw new IllegalStateException("Extension instance(name: " + name + ", class: " + + type + ") could not be instantiated: " + t.getMessage(), t); + } + } + + private T injectExtension(T instance) { + try { + if (objectFactory != null) { + for (Method method : instance.getClass().getMethods()) { + if (method.getName().startsWith("set") + && method.getParameterTypes().length == 1 + && Modifier.isPublic(method.getModifiers())) { + Class pt = method.getParameterTypes()[0]; + try { + String property = method.getName().length() > 3 ? method.getName().substring(3, 4).toLowerCase() + method.getName().substring(4) : ""; + Object object = objectFactory.getExtension(pt, property); + if (object != null) { + method.invoke(instance, object); + } + } catch (Exception e) { + LOGGER.error("fail to inject via method " + method.getName() + + " of interface " + type.getName() + ": " + e.getMessage(), e); + } + } + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + return instance; + } + + private Class getExtensionClass(String name) { + if (type == null) + throw new IllegalArgumentException("Extension type == null"); + if (name == null) + throw new IllegalArgumentException("Extension name == null"); + Class clazz = getExtensionClasses().get(name); + if (clazz == null) + throw new IllegalStateException("No such extension \"" + name + "\" for " + type.getName() + "!"); + return clazz; + } + + private Map> getExtensionClasses() { + Map> classes = cachedClasses.get(); + if (classes == null) { + synchronized (cachedClasses) { + classes = cachedClasses.get(); + if (classes == null) { + classes = loadExtensionClasses(); + cachedClasses.set(classes); + } + } + } + return classes; + } + + // 此方法已经getExtensionClasses方法同步过。 + private Map> loadExtensionClasses() { + final SPI defaultAnnotation = type.getAnnotation(SPI.class); + if (defaultAnnotation != null) { + String value = defaultAnnotation.value(); + if (value != null && (value = value.trim()).length() > 0) { + String[] names = NAME_SEPARATOR.split(value); + if (names.length > 1) { + throw new IllegalStateException("more than 1 default extension name on extension " + type.getName() + + ": " + Arrays.toString(names)); + } + if (names.length == 1) cachedDefaultName = names[0]; + } + } + + Map> extensionClasses = new HashMap>(); + loadFile(extensionClasses, DUBBO_INTERNAL_DIRECTORY); + loadFile(extensionClasses, DUBBO_DIRECTORY); + return extensionClasses; + } + + private void loadFile(Map> extensionClasses, String dir) { + String fileName = dir + type.getName(); + try { + Enumeration configs; + ClassLoader classLoader = findClassLoader(); + if (classLoader != null) { + configs = classLoader.getResources(fileName); + } else { + configs = ClassLoader.getSystemResources(fileName); + } + if (configs != null) { + while (configs.hasMoreElements()) { + java.net.URL config = configs.nextElement(); + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(config.openStream(), "utf-8")); + try { + String line = null; + while ((line = reader.readLine()) != null) { + final int ci = line.indexOf('#'); + if (ci >= 0) line = line.substring(0, ci); + line = line.trim(); + if (line.length() > 0) { + try { + String name = null; + int i = line.indexOf('='); + if (i > 0) { + name = line.substring(0, i).trim(); + line = line.substring(i + 1).trim(); + } + if (line.length() > 0) { + Class clazz = Class.forName(line, true, classLoader); + if (!type.isAssignableFrom(clazz)) { + throw new IllegalStateException("Error when load extension class(interface: " + + type + ", class line: " + clazz.getName() + "), class " + + clazz.getName() + "is not subtype of interface."); + } + if (clazz.isAnnotationPresent(Adaptive.class)) { + if (cachedAdaptiveClass == null) { + cachedAdaptiveClass = clazz; + } else if (!cachedAdaptiveClass.equals(clazz)) { + throw new IllegalStateException("More than 1 adaptive class found: " + + cachedAdaptiveClass.getClass().getName() + + ", " + clazz.getClass().getName()); + } + } else { + try { + clazz.getConstructor(type); + Set> wrappers = cachedWrapperClasses; + if (wrappers == null) { + cachedWrapperClasses = new ConcurrentHashSet>(); + wrappers = cachedWrapperClasses; + } + wrappers.add(clazz); + } catch (NoSuchMethodException e) { + clazz.getConstructor(); + if (name == null || name.length() == 0) { + if (clazz.getSimpleName().length() > type.getSimpleName().length() + && clazz.getSimpleName().endsWith(type.getSimpleName())) { + name = clazz.getSimpleName().substring(0, clazz.getSimpleName().length() - type.getSimpleName().length()).toLowerCase(); + } else { + throw new IllegalStateException("No such extension name for the class " + clazz.getName() + " in the config " + config); + } + } + String[] names = NAME_SEPARATOR.split(name); + if (names != null && names.length > 0) { + Activate activate = clazz.getAnnotation(Activate.class); + if (activate != null) { + cachedActivates.put(names[0], activate); + } + for (String n : names) { + if (!cachedNames.containsKey(clazz)) { + cachedNames.put(clazz, n); + } + Class c = extensionClasses.get(n); + if (c == null) { + extensionClasses.put(n, clazz); + } else if (c != clazz) { + throw new IllegalStateException("Duplicate extension " + type.getName() + " name " + n + " on " + c.getName() + " and " + clazz.getName()); + } + } + } + } + } + } + } catch (Throwable t) { + IllegalStateException e = new IllegalStateException("Failed to load extension class(interface: " + type + ", class line: " + line + ") in " + config + ", cause: " + t.getMessage(), t); + exceptions.put(line, e); + } + } + } // end of while read lines + } finally { + reader.close(); + } + } catch (Throwable t) { + LOGGER.error("Exception when load extension class(interface: " + + type + ", class file: " + config + ") in " + config, t); + } + } // end of while configs + } + } catch (Throwable t) { + LOGGER.error("Exception when load extension class(interface: " + + type + ", description file: " + fileName + ").", t); + } + } + + @SuppressWarnings("unchecked") + private T createAdaptiveExtension() { + try { + return injectExtension((T) getAdaptiveExtensionClass().newInstance()); + } catch (Exception e) { + throw new IllegalStateException("Can not create adaptive extenstion " + type + ", cause: " + e.getMessage(), e); + } + } + + private Class getAdaptiveExtensionClass() { + getExtensionClasses(); + if (cachedAdaptiveClass != null) { + return cachedAdaptiveClass; + } + return cachedAdaptiveClass = createAdaptiveExtensionClass(); + } + + private Class createAdaptiveExtensionClass() { + String code = createAdaptiveExtensionClassCode(); + ClassLoader classLoader = findClassLoader(); + Compiler compiler = ExtensionLoader.getExtensionLoader(Compiler.class).getAdaptiveExtension(); + return compiler.compile(code, classLoader); + } + + private String createAdaptiveExtensionClassCode() { + StringBuilder codeBuidler = new StringBuilder(); + Method[] methods = type.getMethods(); + boolean hasAdaptiveAnnotation = false; + for (Method m : methods) { + if (m.isAnnotationPresent(Adaptive.class)) { + hasAdaptiveAnnotation = true; + break; + } + } + // 完全没有Adaptive方法，则不需要生成Adaptive类 + if (!hasAdaptiveAnnotation) + throw new IllegalStateException("No adaptive method on extension " + type.getName() + ", refuse to create the adaptive class!"); + + codeBuidler.append("package " + type.getPackage().getName() + ";"); + codeBuidler.append("\nimport " + ExtensionLoader.class.getName() + ";"); + codeBuidler.append("\npublic class " + type.getSimpleName() + "$Adpative" + " implements " + type.getCanonicalName() + " {"); + + for (Method method : methods) { + Class rt = method.getReturnType(); + Class[] pts = method.getParameterTypes(); + Class[] ets = method.getExceptionTypes(); + + Adaptive adaptiveAnnotation = method.getAnnotation(Adaptive.class); + StringBuilder code = new StringBuilder(512); + if (adaptiveAnnotation == null) { + code.append("throw new UnsupportedOperationException(\"method ") + .append(method.toString()).append(" of interface ") + .append(type.getName()).append(" is not adaptive method!\");"); + } else { + int configTypeIndex = -1; + for (int i = 0; i < pts.length; ++i) { + if (pts[i].equals(Config.class)) { + configTypeIndex = i; + break; + } + } + // 有类型为URL的参数 + if (configTypeIndex != -1) { + // Null Point check + String s = String.format("\nif (arg%d == null) throw new IllegalArgumentException(\"config == null\");", + configTypeIndex); + code.append(s); + + s = String.format("\n%s config = arg%d;", Config.class.getName(), configTypeIndex); + code.append(s); + } + // 参数没有URL类型 + else { + String attribMethod = null; + + // 找到参数的URL属性 + LBL_PTS: + for (int i = 0; i < pts.length; ++i) { + Method[] ms = pts[i].getMethods(); + for (Method m : ms) { + String name = m.getName(); + if ((name.startsWith("get") || name.length() > 3) + && Modifier.isPublic(m.getModifiers()) + && !Modifier.isStatic(m.getModifiers()) + && m.getParameterTypes().length == 0 + && m.getReturnType() == Config.class) { + configTypeIndex = i; + attribMethod = name; + break LBL_PTS; + } + } + } + if (attribMethod == null) { + throw new IllegalStateException("fail to create adative class for interface " + type.getName() + + ": not found config parameter or config attribute in parameters of method " + method.getName()); + } + + // Null point check + String s = String.format("\nif (arg%d == null) throw new IllegalArgumentException(\"%s argument == null\");", + configTypeIndex, pts[configTypeIndex].getName()); + code.append(s); + s = String.format("\nif (arg%d.%s() == null) throw new IllegalArgumentException(\"%s argument %s() == null\");", + configTypeIndex, attribMethod, pts[configTypeIndex].getName(), attribMethod); + code.append(s); + + s = String.format("%s config = arg%d.%s();", Config.class.getName(), configTypeIndex, attribMethod); + code.append(s); + } + + String[] value = adaptiveAnnotation.value(); + // 没有设置Key，则使用“扩展点接口名的点分隔 作为Key + if (value.length == 0) { + char[] charArray = type.getSimpleName().toCharArray(); + StringBuilder sb = new StringBuilder(128); + for (int i = 0; i < charArray.length; i++) { + if (Character.isUpperCase(charArray[i])) { + if (i != 0) { + sb.append("."); + } + sb.append(Character.toLowerCase(charArray[i])); + } else { + sb.append(charArray[i]); + } + } + value = new String[]{sb.toString()}; + } + + String defaultExtName = cachedDefaultName; + String getNameCode = null; + for (int i = value.length - 1; i >= 0; --i) { + if (i == value.length - 1) { + if (null != defaultExtName) { + getNameCode = String.format("config.getParameter(\"%s\", \"%s\")", value[i], defaultExtName); + } else { + getNameCode = String.format("config.getParameter(\"%s\")", value[i]); + } + } else { + getNameCode = String.format("config.getParameter(\"%s\", %s)", value[i], getNameCode); + } + } + code.append("\nString extName = ").append(getNameCode).append(";"); + // check extName == null? + String s = String.format("\nif(extName == null) " + + "throw new IllegalStateException(\"Fail to get extension(%s) name from config(\" + config.toString() + \") use keys(%s)\");", + type.getName(), Arrays.toString(value)); + code.append(s); + + s = String.format("\n%s extension = (% 0) { + codeBuidler.append(", "); + } + codeBuidler.append(pts[i].getCanonicalName()); + codeBuidler.append(" "); + codeBuidler.append("arg" + i); + } + codeBuidler.append(")"); + if (ets.length > 0) { + codeBuidler.append(" throws "); + for (int i = 0; i < ets.length; i++) { + if (i > 0) { + codeBuidler.append(", "); + } + codeBuidler.append(pts[i].getCanonicalName()); + } + } + codeBuidler.append(" {"); + codeBuidler.append(code.toString()); + codeBuidler.append("\n}"); + } + codeBuidler.append("\n}"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(codeBuidler.toString()); + } + return codeBuidler.toString(); + } + + private static ClassLoader findClassLoader() { + return ExtensionLoader.class.getClassLoader(); + } + + @Override + public String toString() { + return this.getClass().getName() + "[" + type.getName() + "]"; + } + +} + diff --git a/job-core/src/main/java/com/lts/job/core/extension/SPI.java b/job-core/src/main/java/com/lts/job/core/extension/SPI.java new file mode 100644 index 000000000..768ebd7e1 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/SPI.java @@ -0,0 +1,33 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension; + +import java.lang.annotation.*; + +/** + * @author william.liangf + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface SPI { + + /** + * 缺省扩展点名。 + */ + String value() default ""; + +} \ No newline at end of file diff --git a/job-core/src/main/java/com/lts/job/core/extension/factory/AdaptiveExtensionFactory.java b/job-core/src/main/java/com/lts/job/core/extension/factory/AdaptiveExtensionFactory.java new file mode 100644 index 000000000..3257137a7 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/factory/AdaptiveExtensionFactory.java @@ -0,0 +1,55 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension.factory; + +import com.lts.job.core.extension.Adaptive; +import com.lts.job.core.extension.ExtensionFactory; +import com.lts.job.core.extension.ExtensionLoader; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * AdaptiveExtensionFactory + * + * @author william.liangf + */ +@Adaptive +public class AdaptiveExtensionFactory implements ExtensionFactory { + + private final List factories; + + public AdaptiveExtensionFactory() { + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(ExtensionFactory.class); + List list = new ArrayList(); + for (String name : loader.getSupportedExtensions()) { + list.add(loader.getExtension(name)); + } + factories = Collections.unmodifiableList(list); + } + + public T getExtension(Class type, String name) { + for (ExtensionFactory factory : factories) { + T extension = factory.getExtension(type, name); + if (extension != null) { + return extension; + } + } + return null; + } + +} \ No newline at end of file diff --git a/job-core/src/main/java/com/lts/job/core/extension/factory/SpiExtensionFactory.java b/job-core/src/main/java/com/lts/job/core/extension/factory/SpiExtensionFactory.java new file mode 100644 index 000000000..3ca4dee2d --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/extension/factory/SpiExtensionFactory.java @@ -0,0 +1,39 @@ +/* + * Copyright 1999-2012 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lts.job.core.extension.factory; + +import com.lts.job.core.extension.ExtensionFactory; +import com.lts.job.core.extension.ExtensionLoader; +import com.lts.job.core.extension.SPI; + +/** + * SpiExtensionFactory + * + * @author william.liangf + */ +public class SpiExtensionFactory implements ExtensionFactory { + + public T getExtension(Class type, String name) { + if (type.isInterface() && type.isAnnotationPresent(SPI.class)) { + ExtensionLoader loader = ExtensionLoader.getExtensionLoader(type); + if (loader.getSupportedExtensions().size() > 0) { + return loader.getAdaptiveExtension(); + } + } + return null; + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java b/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java index 0f7ecb63d..edc9014fb 100644 --- a/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java +++ b/job-core/src/main/java/com/lts/job/core/factory/JobNodeConfigFactory.java @@ -1,7 +1,7 @@ package com.lts.job.core.factory; import com.lts.job.core.constant.Constants; -import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.core.cluster.Config; import com.lts.job.core.util.StringUtils; /** @@ -9,12 +9,12 @@ */ public class JobNodeConfigFactory { - public static JobNodeConfig getDefaultConfig() { - JobNodeConfig config = new JobNodeConfig(); + public static Config getDefaultConfig() { + Config config = new Config(); config.setIdentity(StringUtils.generateUUID()); config.setWorkThreads(Constants.AVAILABLE_PROCESSOR); config.setNodeGroup("lts"); - config.setRegistryAddress("zookeeper://localhost:2181"); + config.setRegistryAddress("zookeeper://127.0.0.1:2181"); config.setInvokeTimeoutMillis(1000 * 6); config.setListenPort(0); config.setJobInfoSavePath(Constants.USER_HOME); diff --git a/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java b/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java index e3fd87ee0..901a04ebb 100644 --- a/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java +++ b/job-core/src/main/java/com/lts/job/core/factory/NodeFactory.java @@ -1,7 +1,7 @@ package com.lts.job.core.factory; import com.lts.job.core.cluster.Node; -import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.core.cluster.Config; import com.lts.job.core.util.NetUtils; /** @@ -10,7 +10,7 @@ */ public class NodeFactory { - public static T create(Class clazz, JobNodeConfig config) { + public static T create(Class clazz, Config config) { try { T node = clazz.newInstance(); node.setIp(NetUtils.getLocalHost()); diff --git a/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java b/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java index ea33e97da..fc261334d 100644 --- a/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java +++ b/job-core/src/main/java/com/lts/job/core/listener/SelfChangeListener.java @@ -4,7 +4,7 @@ import com.lts.job.core.cluster.Node; import com.lts.job.core.cluster.NodeType; import com.lts.job.core.constant.EcTopic; -import com.lts.job.core.domain.JobNodeConfig; +import com.lts.job.core.cluster.Config; import com.lts.job.core.util.CollectionUtils; import com.lts.job.ec.EventCenter; import com.lts.job.ec.EventInfo; @@ -18,7 +18,7 @@ */ public class SelfChangeListener implements NodeChangeListener { - private JobNodeConfig config; + private Config config; private EventCenter eventCenter; public SelfChangeListener(Application application) { diff --git a/job-core/src/main/java/com/lts/job/core/util/ClassHelper.java b/job-core/src/main/java/com/lts/job/core/util/ClassHelper.java new file mode 100644 index 000000000..3cf18ecdf --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/util/ClassHelper.java @@ -0,0 +1,188 @@ +package com.lts.job.core.util; + +import java.lang.reflect.Array; +import java.util.*; + +/** + * Created by hugui on 5/18/15. + */ +public class ClassHelper { + + public static Class forNameWithThreadContextClassLoader(String name) + throws ClassNotFoundException { + return forName(name, Thread.currentThread().getContextClassLoader()); + } + + public static Class forNameWithCallerClassLoader(String name, Class caller) + throws ClassNotFoundException { + return forName(name, caller.getClassLoader()); + } + + public static ClassLoader getCallerClassLoader(Class caller) { + return caller.getClassLoader(); + } + + /** + * get class loader + * + * @param cls + * @return class loader + */ + public static ClassLoader getClassLoader(Class cls) { + ClassLoader cl = null; + try { + cl = Thread.currentThread().getContextClassLoader(); + } catch (Throwable ex) { + // Cannot access thread context ClassLoader - falling back to system class loader... + } + if (cl == null) { + // No thread context class loader -> use class loader of this class. + cl = cls.getClassLoader(); + } + return cl; + } + + /** + * Return the default ClassLoader to use: typically the thread context + * ClassLoader, if available; the ClassLoader that loaded the ClassUtils + * class will be used as fallback. + *

+ * Call this method if you intend to use the thread context ClassLoader in a + * scenario where you absolutely need a non-null ClassLoader reference: for + * example, for class path resource loading (but not necessarily for + * Class.forName, which accepts a null ClassLoader + * reference as well). + * + * @return the default ClassLoader (never null) + * @see java.lang.Thread#getContextClassLoader() + */ + public static ClassLoader getClassLoader() { + return getClassLoader(ClassHelper.class); + } + + /** + * Same as Class.forName(), except that it works for primitive + * types. + */ + public static Class forName(String name) throws ClassNotFoundException { + return forName(name, getClassLoader()); + } + + /** + * Replacement for Class.forName() that also returns Class + * instances for primitives (like "int") and array class names (like + * "String[]"). + * + * @param name the name of the Class + * @param classLoader the class loader to use (may be null, + * which indicates the default class loader) + * @return Class instance for the supplied name + * @throws ClassNotFoundException if the class was not found + * @throws LinkageError if the class file could not be loaded + * @see Class#forName(String, boolean, ClassLoader) + */ + public static Class forName(String name, ClassLoader classLoader) + throws ClassNotFoundException, LinkageError { + + Class clazz = resolvePrimitiveClassName(name); + if (clazz != null) { + return clazz; + } + + // "java.lang.String[]" style arrays + if (name.endsWith(ARRAY_SUFFIX)) { + String elementClassName = name.substring(0, name.length() - ARRAY_SUFFIX.length()); + Class elementClass = forName(elementClassName, classLoader); + return Array.newInstance(elementClass, 0).getClass(); + } + + // "[Ljava.lang.String;" style arrays + int internalArrayMarker = name.indexOf(INTERNAL_ARRAY_PREFIX); + if (internalArrayMarker != -1 && name.endsWith(";")) { + String elementClassName = null; + if (internalArrayMarker == 0) { + elementClassName = name + .substring(INTERNAL_ARRAY_PREFIX.length(), name.length() - 1); + } else if (name.startsWith("[")) { + elementClassName = name.substring(1); + } + Class elementClass = forName(elementClassName, classLoader); + return Array.newInstance(elementClass, 0).getClass(); + } + + ClassLoader classLoaderToUse = classLoader; + if (classLoaderToUse == null) { + classLoaderToUse = getClassLoader(); + } + return classLoaderToUse.loadClass(name); + } + + /** + * Resolve the given class name as primitive class, if appropriate, + * according to the JVM's naming rules for primitive classes. + *

+ * Also supports the JVM's internal class names for primitive arrays. Does + * not support the "[]" suffix notation for primitive arrays; this is + * only supported by {@link #forName}. + * + * @param name the name of the potentially primitive class + * @return the primitive class, or null if the name does not + * denote a primitive class or primitive array class + */ + public static Class resolvePrimitiveClassName(String name) { + Class result = null; + // Most class names will be quite long, considering that they + // SHOULD sit in a package, so a length check is worthwhile. + if (name != null && name.length() <= 8) { + // Could be a primitive - likely. + result = (Class) primitiveTypeNameMap.get(name); + } + return result; + } + + /** Suffix for array class names: "[]" */ + public static final String ARRAY_SUFFIX = "[]"; + /** Prefix for internal array class names: "[L" */ + private static final String INTERNAL_ARRAY_PREFIX = "[L"; + + /** + * Map with primitive type name as key and corresponding primitive type as + * value, for example: "int" -> "int.class". + */ + private static final Map> primitiveTypeNameMap = new HashMap>(16); + + /** + * Map with primitive wrapper type as key and corresponding primitive type + * as value, for example: Integer.class -> int.class. + */ + private static final Map,Class> primitiveWrapperTypeMap = new HashMap, Class>(8); + + static { + primitiveWrapperTypeMap.put(Boolean.class, boolean.class); + primitiveWrapperTypeMap.put(Byte.class, byte.class); + primitiveWrapperTypeMap.put(Character.class, char.class); + primitiveWrapperTypeMap.put(Double.class, double.class); + primitiveWrapperTypeMap.put(Float.class, float.class); + primitiveWrapperTypeMap.put(Integer.class, int.class); + primitiveWrapperTypeMap.put(Long.class, long.class); + primitiveWrapperTypeMap.put(Short.class, short.class); + + Set> primitiveTypeNames = new HashSet>(16); + primitiveTypeNames.addAll(primitiveWrapperTypeMap.values()); + primitiveTypeNames.addAll(Arrays + .asList(new Class[]{boolean[].class, byte[].class, char[].class, double[].class, + float[].class, int[].class, long[].class, short[].class})); + for (Iterator> it = primitiveTypeNames.iterator(); it.hasNext();) { + Class primitiveClass = (Class) it.next(); + primitiveTypeNameMap.put(primitiveClass.getName(), primitiveClass); + } + } + + public static String toShortString(Object obj){ + if(obj == null){ + return "null"; + } + return obj.getClass().getSimpleName() + "@" + System.identityHashCode(obj); + + } +} \ No newline at end of file diff --git a/job-core/src/main/java/com/lts/job/core/util/Holder.java b/job-core/src/main/java/com/lts/job/core/util/Holder.java new file mode 100644 index 000000000..ff11d0234 --- /dev/null +++ b/job-core/src/main/java/com/lts/job/core/util/Holder.java @@ -0,0 +1,18 @@ +package com.lts.job.core.util; + +/** + * Created by hugui on 5/18/15. + */ +public class Holder { + + private volatile T value; + + public void set(T value) { + this.value = value; + } + + public T get() { + return value; + } + +} diff --git a/job-core/src/main/java/com/lts/job/core/util/StringUtils.java b/job-core/src/main/java/com/lts/job/core/util/StringUtils.java index 86729b162..38e59db5c 100644 --- a/job-core/src/main/java/com/lts/job/core/util/StringUtils.java +++ b/job-core/src/main/java/com/lts/job/core/util/StringUtils.java @@ -3,6 +3,9 @@ import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; +import java.io.PrintWriter; +import java.io.StringWriter; + /** * @author Robert HG (254963746@qq.com) on 7/24/14. */ @@ -74,4 +77,43 @@ public static String format(String format, Object...args){ FormattingTuple ft = MessageFormatter.arrayFormat(format, args); return ft.getMessage(); } + + /** + * + * @param e + * @return string + */ + public static String toString(Throwable e) { + StringWriter w = new StringWriter(); + PrintWriter p = new PrintWriter(w); + p.print(e.getClass().getName()); + if (e.getMessage() != null) { + p.print(": " + e.getMessage()); + } + p.println(); + try { + e.printStackTrace(p); + return w.toString(); + } finally { + p.close(); + } + } + + /** + * + * @param msg + * @param e + * @return string + */ + public static String toString(String msg, Throwable e) { + StringWriter w = new StringWriter(); + w.write(msg + "\n"); + PrintWriter p = new PrintWriter(w); + try { + e.printStackTrace(p); + return w.toString(); + } finally { + p.close(); + } + } } diff --git a/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.compiler.Compiler b/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.compiler.Compiler new file mode 100644 index 000000000..14f4908f9 --- /dev/null +++ b/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.compiler.Compiler @@ -0,0 +1,3 @@ +adaptive=com.lts.job.core.compiler.support.AdaptiveCompiler +jdk=com.lts.job.core.compiler.support.JdkCompiler +javassist=com.lts.job.core.compiler.support.JavassistCompiler \ No newline at end of file diff --git a/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.extension.ExtensionFactory b/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.extension.ExtensionFactory new file mode 100755 index 000000000..e0a245016 --- /dev/null +++ b/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.extension.ExtensionFactory @@ -0,0 +1,2 @@ +adaptive=com.lts.job.core.extension.factory.AdaptiveExtensionFactory +spi=com.lts.job.core.extension.factory.SpiExtensionFactory \ No newline at end of file diff --git a/job-core/src/test/java/com/lts/job/core/FileLockTest.java b/job-core/src/test/java/com/lts/job/core/FileLockTest.java index 19a71d477..957306786 100644 --- a/job-core/src/test/java/com/lts/job/core/FileLockTest.java +++ b/job-core/src/test/java/com/lts/job/core/FileLockTest.java @@ -42,7 +42,7 @@ class Thread_readFile extends Thread { public void run() { try { Long start = System.currentTimeMillis(); - File file = new File("/Users/hugui/.dubbo/dubbo-registry-localhost.cache"); + File file = new File("/Users/hugui/.dubbo/dubbo-registry-127.0.0.1.cache"); //给该文件加锁 RandomAccessFile fis = new RandomAccessFile(file, "rw"); diff --git a/job-core/src/test/java/com/lts/job/core/NetUtilsTest.java b/job-core/src/test/java/com/lts/job/core/NetUtilsTest.java index ce19dbe7d..e16093993 100644 --- a/job-core/src/test/java/com/lts/job/core/NetUtilsTest.java +++ b/job-core/src/test/java/com/lts/job/core/NetUtilsTest.java @@ -12,7 +12,7 @@ public class NetUtilsTest { public void test() { System.out.println(NetUtils.getAvailablePort()); - System.out.println(NetUtils.getIpByHost("localhost")); + System.out.println(NetUtils.getIpByHost("127.0.0.1")); System.out.println(NetUtils.getLocalAddress()); diff --git a/job-core/src/test/java/com/lts/job/core/spi/MainTest.java b/job-core/src/test/java/com/lts/job/core/spi/MainTest.java new file mode 100644 index 000000000..391daa020 --- /dev/null +++ b/job-core/src/test/java/com/lts/job/core/spi/MainTest.java @@ -0,0 +1,23 @@ +package com.lts.job.core.spi; + +import com.lts.job.core.Application; +import com.lts.job.core.cluster.Config; +import com.lts.job.core.extension.ExtensionLoader; + +/** + * Created by hugui on 5/18/15. + */ +public class MainTest { + + public static void main(String[] args) { + +// TestService testService = ExtensionLoader.getExtensionLoader(TestService.class).getExtension("test2"); + TestService testService = ExtensionLoader.getExtensionLoader(TestService.class).getAdaptiveExtension(); + Config config = new Config(); + config.setParameter("test.type", "test2"); + testService.sayHello(config); + Application application = new Application() { + }; + } + +} diff --git a/job-core/src/test/java/com/lts/job/core/spi/TestService.java b/job-core/src/test/java/com/lts/job/core/spi/TestService.java new file mode 100644 index 000000000..ed794bb76 --- /dev/null +++ b/job-core/src/test/java/com/lts/job/core/spi/TestService.java @@ -0,0 +1,16 @@ +package com.lts.job.core.spi; + +import com.lts.job.core.cluster.Config; +import com.lts.job.core.extension.Adaptive; +import com.lts.job.core.extension.SPI; + +/** + * Created by hugui on 5/18/15. + */ +@SPI("test1") +public interface TestService { + + @Adaptive("test.type") + public void sayHello(Config config); + +} diff --git a/job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl.java b/job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl.java new file mode 100644 index 000000000..f39f16234 --- /dev/null +++ b/job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl.java @@ -0,0 +1,14 @@ +package com.lts.job.core.spi; + +import com.lts.job.core.cluster.Config; + +/** + * Created by hugui on 5/18/15. + */ +public class TestServiceImpl implements TestService { + @Override + public void sayHello(Config config) { + System.out.println("1"); + } + +} diff --git a/job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl2.java b/job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl2.java new file mode 100644 index 000000000..a16aad07e --- /dev/null +++ b/job-core/src/test/java/com/lts/job/core/spi/TestServiceImpl2.java @@ -0,0 +1,14 @@ +package com.lts.job.core.spi; + + +import com.lts.job.core.cluster.Config; + +/** + * Created by hugui on 5/18/15. + */ +public class TestServiceImpl2 implements TestService { + @Override + public void sayHello(Config config) { + System.out.println("2"); + } +} diff --git a/job-core/src/test/resources/META-INF/lts/com.lts.job.core.spi.TestService b/job-core/src/test/resources/META-INF/lts/com.lts.job.core.spi.TestService new file mode 100644 index 000000000..ac89502ca --- /dev/null +++ b/job-core/src/test/resources/META-INF/lts/com.lts.job.core.spi.TestService @@ -0,0 +1,2 @@ +test1=com.lts.job.core.spi.TestServiceImpl +test2=com.lts.job.core.spi.TestServiceImpl2 \ No newline at end of file diff --git a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java index 3c5864505..3d84d1678 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java @@ -19,7 +19,7 @@ public static void main(String[] args) throws IOException { // final JobClient jobClient = new JobClient(); jobClient.setNodeGroup("test_jobClient"); // jobClient.setClusterName("lts"); - jobClient.setRegistryAddress("zookeeper://localhost:2181"); + jobClient.setRegistryAddress("zookeeper://127.0.0.1:2181"); // 任务重试保存地址，默认用户目录下 // jobClient.setJobInfoSavePath(Constants.USER_HOME); jobClient.setJobFinishedHandler(new JobFinishedHandlerImpl()); diff --git a/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java b/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java index 3a9b939cb..3a8dc7485 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java @@ -17,7 +17,7 @@ public static void main(String[] args) { final JobTracker jobTracker = new JobTracker(); // 节点信息配置 - jobTracker.setRegistryAddress("zookeeper://localhost:2181"); + jobTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); jobTracker.setListenPort(35002); // 默认 35001 // jobTracker.setClusterName("lts"); @@ -25,7 +25,7 @@ public static void main(String[] args) { // mongo 配置 Config config = new Config(); - config.setAddresses(new String[]{"localhost:27017"}); + config.setAddresses(new String[]{"127.0.0.1:27017"}); config.setUsername("lts"); config.setPassword("lts"); config.setDbName("job"); diff --git a/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java b/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java index f8f3def6d..589b06eb5 100644 --- a/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java @@ -13,7 +13,7 @@ public class TaskTrackerTest { public static void main(String[] args) { final TaskTracker taskTracker = new TaskTracker(); taskTracker.setJobRunnerClass(TestJobRunner.class); - taskTracker.setRegistryAddress("zookeeper://localhost:2181"); + taskTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); taskTracker.setNodeGroup("test_trade_TaskTracker"); // taskTracker.setClusterName("lts"); taskTracker.setWorkThreads(20); diff --git a/job-example/src/main/resources/lts-spring-job-client.xml b/job-example/src/main/resources/lts-spring-job-client.xml index 9e5e03ac6..f073c036b 100644 --- a/job-example/src/main/resources/lts-spring-job-client.xml +++ b/job-example/src/main/resources/lts-spring-job-client.xml @@ -9,7 +9,7 @@ - + diff --git a/job-example/src/main/resources/lts-spring-job-tacker.xml b/job-example/src/main/resources/lts-spring-job-tacker.xml index 376f5cbe3..9c9b22d87 100644 --- a/job-example/src/main/resources/lts-spring-job-tacker.xml +++ b/job-example/src/main/resources/lts-spring-job-tacker.xml @@ -8,7 +8,7 @@ - localhost:27017 + 127.0.0.1:27017 @@ -29,7 +29,7 @@ - + diff --git a/job-example/src/main/resources/lts-spring-job-tasktracker.xml b/job-example/src/main/resources/lts-spring-job-tasktracker.xml index 20c7a960f..0af6fa815 100644 --- a/job-example/src/main/resources/lts-spring-job-tasktracker.xml +++ b/job-example/src/main/resources/lts-spring-job-tasktracker.xml @@ -8,7 +8,7 @@ - + diff --git a/pom.xml b/pom.xml index cf16e8969..033014d41 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,7 @@ 1.1.10 0.1 3.4.5 + 3.19.0-GA @@ -64,7 +65,7 @@ com.netflix.curator curator-framework ${curator.version} - provided + redis.clients @@ -91,6 +92,11 @@ slf4j-log4j12 1.6.1 + + org.javassist + javassist + ${javassist.version} + From daa489c96d7a8392e0750a84ea58c847045570af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E8=B4=B5?= Date: Mon, 18 May 2015 18:44:36 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20loadbalance=20spi?= =?UTF-8?q?=E6=94=AF=E6=8C=81,=20=20=E4=BC=98=E5=8C=96=E5=BF=83=E8=B7=B3?= =?UTF-8?q?=EF=BC=8C=E5=BD=93=E6=B2=A1=E6=9C=89=E6=89=BE=E5=88=B0jobTracke?= =?UTF-8?q?r=20=E6=97=B6=EF=BC=8C=E5=8A=A0=E9=80=9F=E5=BF=83=E8=B7=B3?= =?UTF-8?q?=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/core/cluster/AbstractClientNode.java | 17 +++----- .../core/loadbalance/AbstractLoadBalance.java | 4 +- .../lts/job/core/loadbalance/LoadBalance.java | 8 +++- .../job/core/remoting/HeartBeatMonitor.java | 41 ++++++++++++++----- .../core/remoting/RemotingClientDelegate.java | 7 ++-- .../com.lts.job.core.loadbalance.LoadBalance | 2 + .../java/com/lts/job/core/spi/MainTest.java | 2 +- .../com/lts/job/core/spi/TestService.java | 2 +- .../lts/job/example/api/JobClientTest.java | 2 + .../java/com/lts/job/tracker/JobTracker.java | 13 +++++- .../tracker/domain/JobTrackerApplication.java | 1 - .../job/tracker/support/JobClientManager.java | 12 ++++-- .../tracker/support/TaskTrackerManager.java | 7 +++- 13 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.loadbalance.LoadBalance diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java index 38e6cafca..cce363cc2 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractClientNode.java @@ -2,11 +2,9 @@ import com.lts.job.core.Application; import com.lts.job.core.constant.Constants; -import com.lts.job.core.loadbalance.LoadBalance; -import com.lts.job.core.loadbalance.RandomLoadBalance; +import com.lts.job.core.factory.NamedThreadFactory; import com.lts.job.core.remoting.HeartBeatMonitor; import com.lts.job.core.remoting.RemotingClientDelegate; -import com.lts.job.core.factory.NamedThreadFactory; import com.lts.job.core.util.StringUtils; import com.lts.job.remoting.netty.NettyClientConfig; import com.lts.job.remoting.netty.NettyRemotingClient; @@ -21,16 +19,11 @@ public abstract class AbstractClientNode extends AbstractJobNode { protected RemotingClientDelegate remotingClient; - private LoadBalance serverConnectLoadBalance; private HeartBeatMonitor heartBeatMonitor; protected void innerStart() { - if (serverConnectLoadBalance == null) { - serverConnectLoadBalance = new RandomLoadBalance(); - } - this.remotingClient = new RemotingClientDelegate( - new NettyRemotingClient(getNettyClientConfig()), application, serverConnectLoadBalance); + this.remotingClient = new RemotingClientDelegate(new NettyRemotingClient(getNettyClientConfig()), application); this.heartBeatMonitor = new HeartBeatMonitor(remotingClient, application); remotingClient.start(); @@ -89,10 +82,10 @@ protected NettyClientConfig getNettyClientConfig() { /** * 设置连接JobTracker的负载均衡算法 * - * @param serverConnectLoadBalance + * @param loadBalance 算法 random, consistenthash */ - public void setServerConnectLoadBalance(LoadBalance serverConnectLoadBalance) { - this.serverConnectLoadBalance = serverConnectLoadBalance; + public void setLoadBalance(String loadBalance) { + config.setParameter("loadbalance", loadBalance); } } diff --git a/job-core/src/main/java/com/lts/job/core/loadbalance/AbstractLoadBalance.java b/job-core/src/main/java/com/lts/job/core/loadbalance/AbstractLoadBalance.java index fb8c0f111..aca3c3f7c 100644 --- a/job-core/src/main/java/com/lts/job/core/loadbalance/AbstractLoadBalance.java +++ b/job-core/src/main/java/com/lts/job/core/loadbalance/AbstractLoadBalance.java @@ -1,5 +1,7 @@ package com.lts.job.core.loadbalance; +import com.lts.job.core.cluster.Config; + import java.util.List; /** @@ -8,7 +10,7 @@ public abstract class AbstractLoadBalance implements LoadBalance { @Override - public S select(List shards, String seed) { + public S select(Config config, List shards, String seed) { if (shards == null || shards.size() == 0) { return null; } diff --git a/job-core/src/main/java/com/lts/job/core/loadbalance/LoadBalance.java b/job-core/src/main/java/com/lts/job/core/loadbalance/LoadBalance.java index 080ba086f..b3e0fe13e 100644 --- a/job-core/src/main/java/com/lts/job/core/loadbalance/LoadBalance.java +++ b/job-core/src/main/java/com/lts/job/core/loadbalance/LoadBalance.java @@ -1,12 +1,18 @@ package com.lts.job.core.loadbalance; +import com.lts.job.core.cluster.Config; +import com.lts.job.core.extension.Adaptive; +import com.lts.job.core.extension.SPI; + import java.util.List; /** * Robert HG (254963746@qq.com) on 3/25/15. */ +@SPI("random") public interface LoadBalance { - public S select(List shards, String seed); + @Adaptive("loadbalance") + public S select(Config config, List shards, String seed); } diff --git a/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java b/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java index 27f88618c..05873ae90 100644 --- a/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java +++ b/job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java @@ -5,6 +5,7 @@ import com.lts.job.core.cluster.NodeType; import com.lts.job.core.protocol.JobProtos; import com.lts.job.core.protocol.command.HeartBeatRequest; +import com.lts.job.core.util.CollectionUtils; import com.lts.job.remoting.InvokeCallback; import com.lts.job.remoting.netty.ResponseFuture; import com.lts.job.remoting.protocol.RemotingCommand; @@ -50,17 +51,16 @@ private class HeartBeat implements Runnable { @Override public void run() { try { - List jobTrackers = application.getSubscribedNodeManager().getNodeList(NodeType.JOB_TRACKER); - if (jobTrackers == null) { - return; - } - for (Node jobTracker : jobTrackers) { - // 每个JobTracker 都要发送心跳 - if (beat(remotingClient, jobTracker.getAddress())) { - remotingClient.addJobTracker(jobTracker); - remotingClient.setServerEnable(true); - } else { - remotingClient.removeJobTracker(jobTracker); + boolean serverEnable = check(); + if (!serverEnable) { + // 没有jobTracker可用，加快心跳频率，改为3s一次 + while (!serverEnable) { + try { + Thread.sleep(3000L); // sleep 3s + serverEnable = check(); + } catch (Throwable t) { + LOGGER.error(t.getMessage(), t); + } } } } catch (Throwable t) { @@ -68,6 +68,25 @@ public void run() { } } + private boolean check() { + List jobTrackers = application.getSubscribedNodeManager().getNodeList(NodeType.JOB_TRACKER); + if (CollectionUtils.isEmpty(jobTrackers)) { + return false; + } + boolean serverEnable = false; + for (Node jobTracker : jobTrackers) { + // 每个JobTracker 都要发送心跳 + if (beat(remotingClient, jobTracker.getAddress())) { + remotingClient.addJobTracker(jobTracker); + remotingClient.setServerEnable(true); + serverEnable = true; + } else { + remotingClient.removeJobTracker(jobTracker); + } + } + return serverEnable; + } + /** * 发送心跳 * diff --git a/job-core/src/main/java/com/lts/job/core/remoting/RemotingClientDelegate.java b/job-core/src/main/java/com/lts/job/core/remoting/RemotingClientDelegate.java index 4107db3f4..001f0852d 100644 --- a/job-core/src/main/java/com/lts/job/core/remoting/RemotingClientDelegate.java +++ b/job-core/src/main/java/com/lts/job/core/remoting/RemotingClientDelegate.java @@ -3,6 +3,7 @@ import com.lts.job.core.Application; import com.lts.job.core.cluster.Node; import com.lts.job.core.exception.JobTrackerNotFoundException; +import com.lts.job.core.extension.ExtensionLoader; import com.lts.job.core.loadbalance.LoadBalance; import com.lts.job.remoting.InvokeCallback; import com.lts.job.remoting.exception.RemotingCommandFieldCheckException; @@ -33,10 +34,10 @@ public class RemotingClientDelegate { private boolean serverEnable = false; private List jobTrackers; - public RemotingClientDelegate(NettyRemotingClient remotingClient, Application application, LoadBalance loadBalance) { + public RemotingClientDelegate(NettyRemotingClient remotingClient, Application application) { this.remotingClient = remotingClient; this.application = application; - this.loadBalance = loadBalance; + this.loadBalance = ExtensionLoader.getExtensionLoader(LoadBalance.class).getAdaptiveExtension(); this.jobTrackers = new CopyOnWriteArrayList(); } @@ -44,7 +45,7 @@ public Node getJobTrackerNode() throws JobTrackerNotFoundException { if (jobTrackers.size() == 0) { throw new JobTrackerNotFoundException("no available jobTracker!"); } - return loadBalance.select(jobTrackers, application.getConfig().getIdentity()); + return loadBalance.select(application.getConfig(), jobTrackers, application.getConfig().getIdentity()); } public void start() { diff --git a/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.loadbalance.LoadBalance b/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.loadbalance.LoadBalance new file mode 100644 index 000000000..0a8ad1d0d --- /dev/null +++ b/job-core/src/main/resources/META-INF/lts/internal/com.lts.job.core.loadbalance.LoadBalance @@ -0,0 +1,2 @@ +random=com.lts.job.core.loadbalance.RandomLoadBalance +consistenthash=com.lts.job.core.loadbalance.ConsistentHashLoadBalance \ No newline at end of file diff --git a/job-core/src/test/java/com/lts/job/core/spi/MainTest.java b/job-core/src/test/java/com/lts/job/core/spi/MainTest.java index 391daa020..52460f826 100644 --- a/job-core/src/test/java/com/lts/job/core/spi/MainTest.java +++ b/job-core/src/test/java/com/lts/job/core/spi/MainTest.java @@ -14,7 +14,7 @@ public static void main(String[] args) { // TestService testService = ExtensionLoader.getExtensionLoader(TestService.class).getExtension("test2"); TestService testService = ExtensionLoader.getExtensionLoader(TestService.class).getAdaptiveExtension(); Config config = new Config(); - config.setParameter("test.type", "test2"); +// config.setParameter("test.type", "test2"); testService.sayHello(config); Application application = new Application() { }; diff --git a/job-core/src/test/java/com/lts/job/core/spi/TestService.java b/job-core/src/test/java/com/lts/job/core/spi/TestService.java index ed794bb76..d3cb31f87 100644 --- a/job-core/src/test/java/com/lts/job/core/spi/TestService.java +++ b/job-core/src/test/java/com/lts/job/core/spi/TestService.java @@ -7,7 +7,7 @@ /** * Created by hugui on 5/18/15. */ -@SPI("test1") +@SPI("test2") public interface TestService { @Adaptive("test.type") diff --git a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java index 3d84d1678..b0478fb23 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java @@ -24,6 +24,8 @@ public static void main(String[] args) throws IOException { // jobClient.setJobInfoSavePath(Constants.USER_HOME); jobClient.setJobFinishedHandler(new JobFinishedHandlerImpl()); jobClient.addMasterChangeListener(new MasterChangeListenerImpl()); + jobClient.setLoadBalance("consistenthash"); + jobClient.setJobInfoSavePath("xx"); jobClient.start(); JobClientTest jobClientTest = new JobClientTest(); diff --git a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java index b63884719..8c7ffb98b 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java @@ -24,6 +24,7 @@ public class JobTracker extends AbstractServerNode { public JobTracker() { + config.setNodeGroup(Constants.DEFAULT_NODE_JOB_TRACKER_GROUP); config.setListenPort(Constants.JOB_TRACKER_DEFAULT_LISTEN_PORT); // 添加节点变化监听器 @@ -32,9 +33,9 @@ public JobTracker() { ChannelManager channelManager = new ChannelManager(); application.setChannelManager(channelManager); // JobClient 管理者 - application.setJobClientManager(new JobClientManager(channelManager)); + application.setJobClientManager(new JobClientManager(application)); // TaskTracker 管理者 - application.setTaskTrackerManager(new TaskTrackerManager(channelManager)); + application.setTaskTrackerManager(new TaskTrackerManager(application)); } @Override @@ -100,6 +101,14 @@ public void setJobFeedbackQueue(JobFeedbackQueue jobFeedbackQueue) { application.setJobFeedbackQueue(jobFeedbackQueue); } + /** + * 设置反馈数据给JobClient的负载均衡算法 + * @param loadBalance + */ + public void setLoadBalance(String loadBalance) { + config.setParameter("loadbalance", loadBalance); + } + public void setOldDataHandler(OldDataHandler oldDataHandler) { application.setOldDataHandler(oldDataHandler); } diff --git a/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerApplication.java b/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerApplication.java index 24978ee2c..6e99375db 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerApplication.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerApplication.java @@ -66,7 +66,6 @@ public ChannelManager getChannelManager() { public void setChannelManager(ChannelManager channelManager) { this.channelManager = channelManager; } - public JobClientManager getJobClientManager() { return jobClientManager; } diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java b/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java index 9a210f38d..84af90557 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java @@ -2,6 +2,7 @@ import com.lts.job.core.cluster.Node; import com.lts.job.core.cluster.NodeType; +import com.lts.job.core.extension.ExtensionLoader; import com.lts.job.core.loadbalance.LoadBalance; import com.lts.job.core.loadbalance.RandomLoadBalance; import com.lts.job.core.util.CollectionUtils; @@ -9,6 +10,7 @@ import com.lts.job.tracker.channel.ChannelManager; import com.lts.job.tracker.channel.ChannelWrapper; import com.lts.job.tracker.domain.JobClientNode; +import com.lts.job.tracker.domain.JobTrackerApplication; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +30,12 @@ public class JobClientManager { private LoadBalance loadBalance; private ChannelManager channelManager; + private JobTrackerApplication application; - public JobClientManager(ChannelManager channelManager) { - this.channelManager = channelManager; - this.loadBalance = new RandomLoadBalance(); + public JobClientManager(JobTrackerApplication application) { + this.application = application; + this.channelManager = application.getChannelManager(); + this.loadBalance = ExtensionLoader.getExtensionLoader(LoadBalance.class).getAdaptiveExtension(); } /** @@ -92,7 +96,7 @@ public JobClientNode getAvailableJobClient(String nodeGroup) { while (list.size() > 0) { - JobClientNode jobClientNode = loadBalance.select(list, null); + JobClientNode jobClientNode = loadBalance.select(application.getConfig(), list, null); if (jobClientNode != null && (jobClientNode.getChannel() == null || jobClientNode.getChannel().isClosed())) { ChannelWrapper channel = channelManager.getChannel(jobClientNode.getNodeGroup(), NodeType.CLIENT, jobClientNode.getIdentity()); diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java b/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java index 8784fa2b4..9c6e9496e 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/TaskTrackerManager.java @@ -6,6 +6,7 @@ import com.lts.job.core.util.ConcurrentHashSet; import com.lts.job.tracker.channel.ChannelManager; import com.lts.job.tracker.channel.ChannelWrapper; +import com.lts.job.tracker.domain.JobTrackerApplication; import com.lts.job.tracker.domain.TaskTrackerNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,9 +23,11 @@ public class TaskTrackerManager { // 单例 private final ConcurrentHashMap> NODE_MAP = new ConcurrentHashMap>(); private ChannelManager channelManager; + private JobTrackerApplication application; - public TaskTrackerManager(ChannelManager channelManager) { - this.channelManager = channelManager; + public TaskTrackerManager(JobTrackerApplication application) { + this.application = application; + this.channelManager = application.getChannelManager(); } /** From 9dfdc7601f1b6662631c1484aae471e4676ca175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E8=B4=B5?= Date: Mon, 18 May 2015 23:39:07 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0redis=20=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=E4=B8=AD=E5=BF=83=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/web/support/node/ZkNodeManager.java | 2 +- .../com/lts/job/client/RetryJobClient.java | 2 +- .../lts/job/client/domain/JobClientNode.java | 4 +- .../lts/job/client/domain/ResponseCode.java | 6 +- .../lts/job/core/cluster/AbstractJobNode.java | 8 +- .../java/com/lts/job/core/cluster/Node.java | 4 +- .../com/lts/job/core/cluster/NodeType.java | 2 +- .../com/lts/job/core/constant/Constants.java | 8 + .../job/core/registry/NodeRegistryUtils.java | 57 ++-- .../job/core/registry/RegistryFactory.java | 6 + .../core/registry/redis/RedisRegistry.java | 319 +++++++++++++++--- .../registry/zookeeper/ZookeeperRegistry.java | 24 +- .../java/com/lts/job/core/RegistryTest.java | 29 -- .../lts/job/example/api/JobClientTest.java | 5 +- .../lts/job/example/api/JobTrackerTest.java | 5 +- .../lts/job/example/api/TaskTrackerTest.java | 5 +- .../java/com/lts/job/tracker/JobTracker.java | 4 +- .../job/tracker/channel/ChannelManager.java | 4 +- .../job/tracker/domain/JobTrackerNode.java | 2 +- .../job/tracker/support/JobClientManager.java | 2 +- .../listener/JobNodeChangeListener.java | 4 +- 21 files changed, 366 insertions(+), 136 deletions(-) delete mode 100644 job-core/src/test/java/com/lts/job/core/RegistryTest.java diff --git a/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java b/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java index 145279e1b..acb7494fc 100644 --- a/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java +++ b/job-admin/src/main/java/com/lts/job/web/support/node/ZkNodeManager.java @@ -35,7 +35,7 @@ public Map> getAllNodes(String clusterName) { if (CollectionUtils.isEmpty(nodes)) { List nodeList = new ArrayList(nodes.size()); for (String node : nodes) { - nodeList.add(NodeRegistryUtils.parse(clusterName, node)); + nodeList.add(NodeRegistryUtils.parse(node)); } nodeMap.put(NodeType.valueOf(nodeType), nodeList); } diff --git a/job-client/src/main/java/com/lts/job/client/RetryJobClient.java b/job-client/src/main/java/com/lts/job/client/RetryJobClient.java index 567d218fc..a0ad670f8 100644 --- a/job-client/src/main/java/com/lts/job/client/RetryJobClient.java +++ b/job-client/src/main/java/com/lts/job/client/RetryJobClient.java @@ -14,7 +14,7 @@ * @author Robert HG (254963746@qq.com) on 8/14/14. * 重试 客户端, 如果 没有可用的JobTracker, 那么存文件, 定时重试 */ -public class RetryJobClient extends JobClient{ +public class RetryJobClient extends JobClient { private RetryScheduler retryScheduler; diff --git a/job-client/src/main/java/com/lts/job/client/domain/JobClientNode.java b/job-client/src/main/java/com/lts/job/client/domain/JobClientNode.java index 6abd67341..23378052a 100644 --- a/job-client/src/main/java/com/lts/job/client/domain/JobClientNode.java +++ b/job-client/src/main/java/com/lts/job/client/domain/JobClientNode.java @@ -10,9 +10,9 @@ public class JobClientNode extends Node { public JobClientNode() { - this.setNodeType(NodeType.CLIENT); + this.setNodeType(NodeType.JOB_CLIENT); this.addListenNodeType(NodeType.JOB_TRACKER); - this.addListenNodeType(NodeType.CLIENT); + this.addListenNodeType(NodeType.JOB_CLIENT); } } diff --git a/job-client/src/main/java/com/lts/job/client/domain/ResponseCode.java b/job-client/src/main/java/com/lts/job/client/domain/ResponseCode.java index 4d6cf17a7..c1eeaee7c 100644 --- a/job-client/src/main/java/com/lts/job/client/domain/ResponseCode.java +++ b/job-client/src/main/java/com/lts/job/client/domain/ResponseCode.java @@ -9,12 +9,12 @@ public class ResponseCode { private ResponseCode(){} // 没有找到 JobTracker 节点 - public static final String JOB_TRACKER_NOT_FOUND = "JOB_TRACKER_NOT_FOUND"; + public static final String JOB_TRACKER_NOT_FOUND = "11"; // 提交失败并且写入文件 - public static final String FAILED_AND_SAVE_FILE = "FAILED_AND_SAVE_FILE"; + public static final String FAILED_AND_SAVE_FILE = "12"; // 请求参数检查失败 - public static final String REQUEST_FILED_CHECK_ERROR = "REQUEST_FILED_CHECK_ERROR"; + public static final String REQUEST_FILED_CHECK_ERROR = "13"; } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java index c5000a355..50183f505 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/AbstractJobNode.java @@ -55,8 +55,6 @@ final public void start() { node = NodeFactory.create(getNodeClass(), config); config.setNodeType(node.getNodeType()); - initRegistry(); - LOGGER.info("当前节点配置:{}", config); // 监听节点 启用/禁用消息 @@ -75,6 +73,8 @@ public void onObserved(EventInfo eventInfo) { innerStart(); + initRegistry(); + registry.register(node); LOGGER.info("启动成功!"); @@ -107,7 +107,9 @@ public void destroy() { private void initRegistry() { registry = RegistryFactory.getRegistry(application); - ((AbstractRegistry) registry).setNode(node); + if (registry instanceof AbstractRegistry) { + ((AbstractRegistry) registry).setNode(node); + } // 订阅的node管理 SubscribedNodeManager subscribedNodeManager = new SubscribedNodeManager(application); application.setSubscribedNodeManager(subscribedNodeManager); diff --git a/job-core/src/main/java/com/lts/job/core/cluster/Node.java b/job-core/src/main/java/com/lts/job/core/cluster/Node.java index 21b29252c..114cbe9d4 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/Node.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/Node.java @@ -127,14 +127,14 @@ public String getAddress() { @Override public String toString() { return "Node{" + - ", isAvailable=" + isAvailable + + "identity='" + identity + '\'' + ", nodeType=" + nodeType + ", ip='" + ip + '\'' + ", port=" + port + ", group='" + group + '\'' + ", createTime=" + createTime + ", threads=" + threads + - ", identity='" + identity + '\'' + + ", isAvailable=" + isAvailable + ", listenNodeTypes=" + listenNodeTypes + '}'; } diff --git a/job-core/src/main/java/com/lts/job/core/cluster/NodeType.java b/job-core/src/main/java/com/lts/job/core/cluster/NodeType.java index 449419612..f158a9399 100644 --- a/job-core/src/main/java/com/lts/job/core/cluster/NodeType.java +++ b/job-core/src/main/java/com/lts/job/core/cluster/NodeType.java @@ -10,5 +10,5 @@ public enum NodeType { // task tracker TASK_TRACKER, // client - CLIENT + JOB_CLIENT } diff --git a/job-core/src/main/java/com/lts/job/core/constant/Constants.java b/job-core/src/main/java/com/lts/job/core/constant/Constants.java index b50f66cf8..c6fddfb3e 100644 --- a/job-core/src/main/java/com/lts/job/core/constant/Constants.java +++ b/job-core/src/main/java/com/lts/job/core/constant/Constants.java @@ -51,4 +51,12 @@ public interface Constants { public static final int DEFAULT_REGISTRY_RETRY_PERIOD = 5 * 1000; public static final Pattern COMMA_SPLIT_PATTERN = Pattern.compile("\\s*[,]+\\s*"); + + /** + * 注册中心自动重连时间 + */ + public static final String REGISTRY_RECONNECT_PERIOD_KEY = "reconnect.period"; + + public static final int DEFAULT_REGISTRY_RECONNECT_PERIOD = 3 * 1000; + } diff --git a/job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java b/job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java index 79f5ee4e0..a51593696 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java +++ b/job-core/src/main/java/com/lts/job/core/registry/NodeRegistryUtils.java @@ -2,13 +2,16 @@ import com.lts.job.core.cluster.Node; import com.lts.job.core.cluster.NodeType; +import com.lts.job.core.util.NetUtils; +import com.lts.job.core.util.StringUtils; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.Date; /** * @author Robert HG (254963746@qq.com) on 5/11/15. *

+ * /LTS/{集群名字}/NODES/TASK_TRACKER/TASK_TRACKER:\\192.168.0.150:8888?group=TASK_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER + * /LTS/{集群名字}/NODES/JOB_CLIENT/JOB_CLIENT:\\192.168.0.150:8888?group=JOB_CLIENT&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER * /LTS/{集群名字}/NODES/JOB_TRACKER/JOB_TRACKER:\\192.168.0.150:8888?group=JOB_TRACKER&threads=8&identity=85750db6-e854-4eb3-a595-9227a5f2c8f6&createTime=1408189898185&isAvailable=true&listenNodeTypes=CLIENT,TASK_TRACKER *

*/ @@ -22,13 +25,14 @@ public static String getNodeTypePath(String clusterName, NodeType nodeType) { return NodeRegistryUtils.getRootPath(clusterName) + "/" + nodeType; } - public static Node parse(String clusterName, String fullPath) { + public static Node parse(String fullPath) { Node node = new Node(); - String nodeType = getMatcher(getRootPath(clusterName) + "/(.*)/", fullPath); - node.setNodeType(NodeType.valueOf(nodeType)); - - String url = getMatcher(getRootPath(clusterName) + "/" + nodeType + "/" + nodeType + ":\\\\\\\\(.*)", fullPath); + String[] nodeDir = fullPath.split("/"); + NodeType nodeType = NodeType.valueOf(nodeDir[4]); + node.setNodeType(nodeType); + String url = nodeDir[5]; + url = url.substring(nodeType.name().length() + 3); String address = url.split("\\?")[0]; String ip = address.split(":")[0]; @@ -56,11 +60,6 @@ public static Node parse(String clusterName, String fullPath) { node.setCreateTime(Long.valueOf(value)); } else if ("isAvailable".equals(key)) { node.setAvailable(Boolean.valueOf(value)); - } else if ("listenNodeTypes".equals(key)) { - String[] nodeTypes = value.split(","); - for (String type : nodeTypes) { - node.addListenNodeType(NodeType.valueOf(type)); - } } } return node; @@ -68,6 +67,7 @@ public static Node parse(String clusterName, String fullPath) { public static String getFullPath(String clusterName, Node node) { StringBuilder path = new StringBuilder(); + path.append(getRootPath(clusterName)) .append("/") .append(node.getNodeType()) @@ -95,22 +95,27 @@ public static String getFullPath(String clusterName, Node node) { .append("&isAvailable=") .append(node.isAvailable()); - if (node.getListenNodeTypes() != null && node.getListenNodeTypes().size() != 0) { - path.append("&listenNodeTypes="); - for (NodeType nodeType : node.getListenNodeTypes()) { - path.append(nodeType).append(","); - } - path.deleteCharAt(path.length() - 1); - } return path.toString(); } - private static String getMatcher(String regex, String source) { - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(source); - while (matcher.find()) { - return matcher.group(1);//只取第一组 - } - return ""; + public static void main(String[] args) { + Node node = new Node(); + node.setGroup("group1"); + node.setIdentity(StringUtils.generateUUID()); + node.setThreads(222); + node.setNodeType(NodeType.JOB_TRACKER); + node.setCreateTime(new Date().getTime()); + node.setPort(2313); + node.setIp(NetUtils.getLocalHost()); + String fullPath = NodeRegistryUtils.getFullPath("lts", node); + System.out.println(fullPath); + + node = NodeRegistryUtils.parse(fullPath); + node.setNodeType(NodeType.JOB_CLIENT); + fullPath = NodeRegistryUtils.getFullPath("lts", node); + System.out.println(fullPath); + + node = NodeRegistryUtils.parse(fullPath); + System.out.println(node); } } diff --git a/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java b/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java index d4b3a8cdd..2a9f4d180 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java +++ b/job-core/src/main/java/com/lts/job/core/registry/RegistryFactory.java @@ -16,8 +16,14 @@ public static Registry getRegistry(Application application) { throw new IllegalArgumentException("address is null！"); } if (address.startsWith("zookeeper://")) { + application.getConfig().setRegistryAddress( + address.replace("zookeeper://", "") + ); return new ZookeeperRegistry(application); } else if (address.startsWith("redis://")) { + application.getConfig().setRegistryAddress( + address.replace("redis://", "") + ); return new RedisRegistry(application); } throw new IllegalArgumentException("illegal address protocol"); diff --git a/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java index 52190bcb5..64d79d6e3 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java +++ b/job-core/src/main/java/com/lts/job/core/registry/redis/RedisRegistry.java @@ -1,21 +1,27 @@ package com.lts.job.core.registry.redis; import com.lts.job.core.Application; +import com.lts.job.core.cluster.Config; import com.lts.job.core.cluster.Node; +import com.lts.job.core.cluster.NodeType; import com.lts.job.core.constant.Constants; import com.lts.job.core.exception.NodeRegistryException; import com.lts.job.core.factory.NamedThreadFactory; import com.lts.job.core.registry.FailbackRegistry; import com.lts.job.core.registry.NodeRegistryUtils; +import com.lts.job.core.registry.NotifyEvent; import com.lts.job.core.registry.NotifyListener; +import com.lts.job.core.util.CollectionUtils; import org.apache.commons.pool.impl.GenericObjectPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPubSub; -import java.util.Map; +import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; /** * @author Robert HG (254963746@qq.com) on 5/17/15. @@ -32,30 +38,36 @@ public class RedisRegistry extends FailbackRegistry { private final ScheduledFuture expireFuture; private final int expirePeriod; private boolean replicate; + private final int reconnectPeriod; + private final ConcurrentMap notifiers = new ConcurrentHashMap(); + + private Config config; public RedisRegistry(Application application) { super(application); + this.config = application.getConfig(); + this.clusterName = config.getClusterName(); - this.clusterName = application.getConfig().getClusterName(); - - GenericObjectPool.Config config = new GenericObjectPool.Config(); + GenericObjectPool.Config redisConfig = new GenericObjectPool.Config(); // TODO 可以设置n多参数 - String address = application.getConfig().getRegistryAddress(); + String address = config.getRegistryAddress(); - String cluster = application.getConfig().getParameter("cluster", "failover"); + String cluster = config.getParameter("cluster", "failover"); if (!"failover".equals(cluster) && !"replicate".equals(cluster)) { throw new IllegalArgumentException("Unsupported redis cluster: " + cluster + ". The redis cluster only supported failover or replicate."); } replicate = "replicate".equals(cluster); + this.reconnectPeriod = config.getParameter(Constants.REGISTRY_RECONNECT_PERIOD_KEY, Constants.DEFAULT_REGISTRY_RECONNECT_PERIOD); + int i = address.indexOf(':'); String host = address.substring(0, i); int port = Integer.parseInt(address.substring(i + 1)); - this.jedisPools.put(address, new JedisPool(config, host, port, + this.jedisPools.put(address, new JedisPool(redisConfig, host, port, Constants.DEFAULT_TIMEOUT)); - this.expirePeriod = application.getConfig().getParameter(Constants.SESSION_TIMEOUT_KEY, Constants.DEFAULT_SESSION_TIMEOUT); + this.expirePeriod = config.getParameter(Constants.SESSION_TIMEOUT_KEY, Constants.DEFAULT_SESSION_TIMEOUT); this.expireFuture = expireExecutor.scheduleWithFixedDelay(new Runnable() { public void run() { @@ -70,35 +82,32 @@ public void run() { private void deferExpired() { for (Map.Entry entry : jedisPools.entrySet()) { -// JedisPool jedisPool = entry.getValue(); -// try { -// Jedis jedis = jedisPool.getResource(); -// try { -// for (URL url : new HashSet(getRegistered())) { -// if (url.getParameter(Constants.DYNAMIC_KEY, true)) { -// String key = toCategoryPath(url); -// if (jedis.hset(key, url.toFullString(), String.valueOf(System.currentTimeMillis() + expirePeriod)) == 1) { -// jedis.publish(key, Constants.REGISTER); -// } -// } -// } -// if (admin) { -// clean(jedis); -// } -// if (! replicate) { -// break;//  如果服务器端已同步数据，只需写入单台机器 -// } -// } finally { -// jedisPool.returnResource(jedis); -// } -// } catch (Throwable t) { -// logger.warn("Failed to write provider heartbeat to redis registry. registry: " + entry.getKey() + ", cause: " + t.getMessage(), t); -// } + JedisPool jedisPool = entry.getValue(); + try { + Jedis jedis = jedisPool.getResource(); + try { + for (Node node : new HashSet(getRegistered())) { + String fullPath = NodeRegistryUtils.getFullPath(clusterName, node); + String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType()); + if (jedis.hset(key, fullPath, String.valueOf(System.currentTimeMillis() + expirePeriod)) == 1) { + jedis.publish(key, Constants.REGISTER); + } + } + if (!replicate) { + break;//  如果服务器端已同步数据，只需写入单台机器 + } + } finally { + jedisPool.returnResource(jedis); + } + } catch (Throwable t) { + LOGGER.warn("Failed to write provider heartbeat to redis registry. registry: " + entry.getKey() + ", cause: " + t.getMessage(), t); + } } } + @Override - public void register(Node node) { + protected void doRegister(Node node) { String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType()); String value = NodeRegistryUtils.getFullPath(clusterName, node); String expire = String.valueOf(System.currentTimeMillis() + expirePeriod); @@ -129,11 +138,10 @@ public void register(Node node) { throw exception; } } - } @Override - public void unregister(Node node) { + protected void doUnRegister(Node node) { String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType()); String value = NodeRegistryUtils.getFullPath(clusterName, node); boolean success = false; @@ -166,23 +174,252 @@ public void unregister(Node node) { } @Override - protected void doRegister(Node node) { + protected void doSubscribe(Node node, NotifyListener listener) { + + List listenNodeTypes = node.getListenNodeTypes(); + if (CollectionUtils.isEmpty(listenNodeTypes)) { + return; + } + for (NodeType listenNodeType : listenNodeTypes) { + String listenNodePath = NodeRegistryUtils.getNodeTypePath(clusterName, listenNodeType); + + Notifier notifier = notifiers.get(listenNodePath); + if (notifier == null) { + Notifier newNotifier = new Notifier(listenNodePath); + notifiers.putIfAbsent(listenNodePath, newNotifier); + notifier = notifiers.get(listenNodePath); + if (notifier == newNotifier) { + notifier.start(); + } + } + + boolean success = false; + NodeRegistryException exception = null; + for (Map.Entry entry : jedisPools.entrySet()) { + JedisPool jedisPool = entry.getValue(); + try { + Jedis jedis = jedisPool.getResource(); + try { + doNotify(jedis, jedis.keys(listenNodePath), Arrays.asList(listener), NotifyEvent.ADD); + success = true; + break; // 只需读一个服务器的数据 + } finally { + jedisPool.returnResource(jedis); + } + } catch (Throwable t) { + exception = new NodeRegistryException("Failed to unregister node to redis registry. registry: " + entry.getKey() + ", node: " + node + ", cause: " + t.getMessage(), t); + } + } + if (exception != null) { + if (success) { + LOGGER.warn(exception.getMessage(), exception); + } else { + throw exception; + } + } + + } } @Override - protected void doUnRegister(Node node) { - + protected void doUnsubscribe(Node node, NotifyListener listener) { } @Override - protected void doSubscribe(Node node, NotifyListener listener) { + public void destroy() { + super.destroy(); + try { + expireFuture.cancel(true); + } catch (Throwable t) { + LOGGER.warn(t.getMessage(), t); + } + try { + for (Notifier notifier : notifiers.values()) { + notifier.shutdown(); + } + } catch (Throwable t) { + LOGGER.warn(t.getMessage(), t); + } + for (Map.Entry entry : jedisPools.entrySet()) { + JedisPool jedisPool = entry.getValue(); + try { + jedisPool.destroy(); + } catch (Throwable t) { + LOGGER.warn("Failed to destroy the redis registry client. registry: " + entry.getKey() + ", cause: " + t.getMessage(), t); + } + } + } + + private void doNotify(Jedis jedis, Collection keys, Collection listeners, NotifyEvent event) { + if (keys == null || keys.size() == 0 + || listeners == null || listeners.size() == 0) { + return; + } + List result = new ArrayList(); + for (String key : keys) { + + Map values = jedis.hgetAll(key); + if (values != null && values.size() > 0) { + for (Map.Entry entry : values.entrySet()) { + Node node = NodeRegistryUtils.parse(entry.getKey()); + result.add(node); + } + } + } + if (result == null || result.size() == 0) { + return; + } + for (NotifyListener listener : listeners) { + notify(event, result, listener); + } } - @Override - protected void doUnsubscribe(Node node, NotifyListener listener) { + private void doNotify(Jedis jedis, String key, NotifyEvent event) { + for (Map.Entry> entry : new HashMap>(getSubscribed()).entrySet()) { + doNotify(jedis, Arrays.asList(key), new HashSet(entry.getValue()), event); + } + } + + private class NotifySub extends JedisPubSub { + + private final JedisPool jedisPool; + + public NotifySub(JedisPool jedisPool) { + this.jedisPool = jedisPool; + } + + @Override + public void onMessage(String key, String msg) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("redis event: " + key + " = " + msg); + } + if (msg.equals(Constants.REGISTER) + || msg.equals(Constants.UNREGISTER)) { + try { + Jedis jedis = jedisPool.getResource(); + try { + NotifyEvent event = msg.equals(Constants.REGISTER) ? NotifyEvent.ADD : NotifyEvent.REMOVE; + doNotify(jedis, key, event); + } finally { + jedisPool.returnResource(jedis); + } + } catch (Throwable t) { // TODO 通知失败没有恢复机制保障 + LOGGER.error(t.getMessage(), t); + } + } + } + + @Override + public void onPMessage(String pattern, String key, String msg) { + onMessage(key, msg); + } + + @Override + public void onSubscribe(String key, int num) { + } + + @Override + public void onPSubscribe(String pattern, int num) { + } + + @Override + public void onUnsubscribe(String key, int num) { + } + + @Override + public void onPUnsubscribe(String pattern, int num) { + } } + private class Notifier extends Thread { + + private final String listenNodePath; + + private volatile Jedis jedis; + + private volatile boolean running = true; + + private final AtomicInteger connectSkip = new AtomicInteger(); + + private final AtomicInteger connectSkiped = new AtomicInteger(); + + private final Random random = new Random(); + + private volatile int connectRandom; + + private void resetSkip() { + connectSkip.set(0); + connectSkiped.set(0); + connectRandom = 0; + } + + private boolean isSkip() { + int skip = connectSkip.get(); // 跳过次数增长 + if (skip >= 10) { // 如果跳过次数增长超过10，取随机数 + if (connectRandom == 0) { + connectRandom = random.nextInt(10); + } + skip = 10 + connectRandom; + } + if (connectSkiped.getAndIncrement() < skip) { // 检查跳过次数 + return true; + } + connectSkip.incrementAndGet(); + connectSkiped.set(0); + connectRandom = 0; + return false; + } + + public Notifier(String listenNodePath) { + super.setDaemon(true); + super.setName("LTSRedisSubscribe"); + this.listenNodePath = listenNodePath; + } + + @Override + public void run() { + while (running) { + try { + if (!isSkip()) { + try { + for (Map.Entry entry : jedisPools.entrySet()) { + JedisPool jedisPool = entry.getValue(); + try { + jedis = jedisPool.getResource(); + try { + jedis.psubscribe(new NotifySub(jedisPool), listenNodePath); // 阻塞 + break; + } finally { + jedisPool.returnBrokenResource(jedis); + } + } catch (Throwable t) { // 重试另一台 + LOGGER.warn("Failed to subscribe node from redis registry. registry: " + entry.getKey() + ", cause: " + t.getMessage(), t); + // 如果在单台redis的情况下，需要休息一会，避免空转占用过多cpu资源 + sleep(reconnectPeriod); + } + } + } catch (Throwable t) { + LOGGER.error(t.getMessage(), t); + sleep(reconnectPeriod); + } + } + } catch (Throwable t) { + LOGGER.error(t.getMessage(), t); + } + } + } + + public void shutdown() { + try { + running = false; + jedis.disconnect(); + } catch (Throwable t) { + LOGGER.warn(t.getMessage(), t); + } + } + + } } diff --git a/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java b/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java index 257d04605..b1a35ac3f 100644 --- a/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java +++ b/job-core/src/main/java/com/lts/job/core/registry/zookeeper/ZookeeperRegistry.java @@ -26,7 +26,7 @@ public class ZookeeperRegistry extends FailbackRegistry { private ZookeeperClient zkClient; // 用来记录父节点下的子节点的变化 - private final ConcurrentHashMap> childrenNodeMap; + private final ConcurrentHashMap> cachedChildrenNodeMap; private final ConcurrentMap> zkListeners; @@ -35,10 +35,9 @@ public class ZookeeperRegistry extends FailbackRegistry { public ZookeeperRegistry(Application application) { super(application); this.clusterName = application.getConfig().getClusterName(); - this.childrenNodeMap = new ConcurrentHashMap>(); + this.cachedChildrenNodeMap = new ConcurrentHashMap>(); - String address = application.getConfig().getRegistryAddress().replace("zookeeper://", ""); - this.zkClient = new ZkClientZookeeperClient(address); + this.zkClient = new ZkClientZookeeperClient(application.getConfig().getRegistryAddress()); this.zkListeners = new ConcurrentHashMap>(); zkClient.addStateListener(new StateListener() { @Override @@ -81,18 +80,16 @@ protected void doSubscribe(Node node, NotifyListener listener) { ChildListener zkListener = addZkListener(node, listener); // 为自己关注的 节点 添加监听 - zkClient.addChildListener(listenNodePath, zkListener); + List children = zkClient.addChildListener(listenNodePath, zkListener); - // 将自己关注的 节点类型加入到节点管理中去 - List children = zkClient.getChildren(listenNodePath); if (CollectionUtils.isNotEmpty(children)) { List listenedNodes = new ArrayList(); for (String child : children) { - Node listenedNode = NodeRegistryUtils.parse(clusterName, listenNodePath + "/" + child); + Node listenedNode = NodeRegistryUtils.parse(listenNodePath + "/" + child); listenedNodes.add(listenedNode); } notify(NotifyEvent.ADD, listenedNodes, listener); - childrenNodeMap.put(listenNodePath, children); + cachedChildrenNodeMap.put(listenNodePath, children); } } } @@ -129,7 +126,8 @@ private ChildListener addZkListener(Node node, final NotifyListener listener) { listeners.putIfAbsent(listener, new ChildListener() { public void childChanged(String parentPath, List currentChilds) { - List oldChilds = childrenNodeMap.get(parentPath); + + List oldChilds = cachedChildrenNodeMap.get(parentPath); // 1. 找出增加的 节点 List addChilds = CollectionUtils.getLeftDiff(currentChilds, oldChilds); // 2. 找出减少的 节点 @@ -139,7 +137,7 @@ public void childChanged(String parentPath, List currentChilds) { List nodes = new ArrayList(addChilds.size()); for (String child : addChilds) { - Node node = NodeRegistryUtils.parse(clusterName, parentPath + "/" + child); + Node node = NodeRegistryUtils.parse(parentPath + "/" + child); nodes.add(node); } ZookeeperRegistry.this.notify(NotifyEvent.ADD, nodes, listener); @@ -148,12 +146,12 @@ public void childChanged(String parentPath, List currentChilds) { if (CollectionUtils.isNotEmpty(decChilds)) { List nodes = new ArrayList(addChilds.size()); for (String child : decChilds) { - Node node = NodeRegistryUtils.parse(clusterName, parentPath + "/" + child); + Node node = NodeRegistryUtils.parse(parentPath + "/" + child); nodes.add(node); } ZookeeperRegistry.this.notify(NotifyEvent.REMOVE, nodes, listener); } - childrenNodeMap.put(parentPath, currentChilds); + cachedChildrenNodeMap.put(parentPath, currentChilds); } }); zkListener = listeners.get(listener); diff --git a/job-core/src/test/java/com/lts/job/core/RegistryTest.java b/job-core/src/test/java/com/lts/job/core/RegistryTest.java deleted file mode 100644 index fb814ad4e..000000000 --- a/job-core/src/test/java/com/lts/job/core/RegistryTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.lts.job.core; - -import com.lts.job.core.cluster.Node; -import com.lts.job.core.cluster.NodeType; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author Robert HG (254963746@qq.com) on 7/23/14. - */ -public class RegistryTest { - - @Test - public void test_registry() throws IOException { - - } - - public class JobClientNode extends Node { - - public JobClientNode() { - super(); - this.setNodeType(NodeType.CLIENT); - this.addListenNodeType(NodeType.JOB_TRACKER); - } - - } - -} diff --git a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java index b0478fb23..27bbb0a37 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobClientTest.java @@ -18,8 +18,9 @@ public static void main(String[] args) throws IOException { JobClient jobClient = new RetryJobClient(); // final JobClient jobClient = new JobClient(); jobClient.setNodeGroup("test_jobClient"); -// jobClient.setClusterName("lts"); - jobClient.setRegistryAddress("zookeeper://127.0.0.1:2181"); + jobClient.setClusterName("test_cluster"); +// jobClient.setRegistryAddress("zookeeper://127.0.0.1:2181"); + jobClient.setRegistryAddress("redis://127.0.0.1:6379"); // 任务重试保存地址，默认用户目录下 // jobClient.setJobInfoSavePath(Constants.USER_HOME); jobClient.setJobFinishedHandler(new JobFinishedHandlerImpl()); diff --git a/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java b/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java index 3a8dc7485..b49d9998b 100644 --- a/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java @@ -17,9 +17,10 @@ public static void main(String[] args) { final JobTracker jobTracker = new JobTracker(); // 节点信息配置 - jobTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); +// jobTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); + jobTracker.setRegistryAddress("redis://127.0.0.1:6379"); jobTracker.setListenPort(35002); // 默认 35001 -// jobTracker.setClusterName("lts"); + jobTracker.setClusterName("test_cluster"); jobTracker.addMasterChangeListener(new MasterChangeListenerImpl()); diff --git a/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java b/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java index 589b06eb5..7a0c9d47c 100644 --- a/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java +++ b/job-example/src/main/java/com/lts/job/example/api/TaskTrackerTest.java @@ -13,9 +13,10 @@ public class TaskTrackerTest { public static void main(String[] args) { final TaskTracker taskTracker = new TaskTracker(); taskTracker.setJobRunnerClass(TestJobRunner.class); - taskTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); +// taskTracker.setRegistryAddress("zookeeper://127.0.0.1:2181"); + taskTracker.setRegistryAddress("redis://127.0.0.1:6379"); taskTracker.setNodeGroup("test_trade_TaskTracker"); -// taskTracker.setClusterName("lts"); + taskTracker.setClusterName("test_cluster"); taskTracker.setWorkThreads(20); // taskTracker.setJobInfoSavePath(Constants.USER_HOME); taskTracker.addMasterChangeListener(new MasterChangeListenerImpl()); diff --git a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java index 8c7ffb98b..e0ce184a0 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/JobTracker.java @@ -24,8 +24,6 @@ public class JobTracker extends AbstractServerNode { public JobTracker() { - - config.setNodeGroup(Constants.DEFAULT_NODE_JOB_TRACKER_GROUP); config.setListenPort(Constants.JOB_TRACKER_DEFAULT_LISTEN_PORT); // 添加节点变化监听器 addNodeChangeListener(new JobNodeChangeListener(application)); @@ -112,4 +110,6 @@ public void setLoadBalance(String loadBalance) { public void setOldDataHandler(OldDataHandler oldDataHandler) { application.setOldDataHandler(oldDataHandler); } + + } diff --git a/job-tracker/src/main/java/com/lts/job/tracker/channel/ChannelManager.java b/job-tracker/src/main/java/com/lts/job/tracker/channel/ChannelManager.java index 92c39ee63..11c5f8f46 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/channel/ChannelManager.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/channel/ChannelManager.java @@ -74,7 +74,7 @@ private void checkCloseChannel(ConcurrentHashMap> c } public List getChannels(String nodeGroup, NodeType nodeType) { - if (nodeType == NodeType.CLIENT) { + if (nodeType == NodeType.JOB_CLIENT) { return clientChannelMap.get(nodeGroup); } else if (nodeType == NodeType.TASK_TRACKER) { return taskTrackerChannelMap.get(nodeGroup); @@ -113,7 +113,7 @@ public void offerChannel(ChannelWrapper channel) { List channels = getChannels(nodeGroup, nodeType); if (channels == null) { channels = new ArrayList(); - if (nodeType == NodeType.CLIENT) { + if (nodeType == NodeType.JOB_CLIENT) { clientChannelMap.put(nodeGroup, channels); } else if (nodeType == NodeType.TASK_TRACKER) { taskTrackerChannelMap.put(nodeGroup, channels); diff --git a/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerNode.java b/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerNode.java index a4351e5e5..7282ad082 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerNode.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerNode.java @@ -11,7 +11,7 @@ public class JobTrackerNode extends Node { public JobTrackerNode() { this.setNodeType(NodeType.JOB_TRACKER); - this.addListenNodeType(NodeType.CLIENT); + this.addListenNodeType(NodeType.JOB_CLIENT); this.addListenNodeType(NodeType.TASK_TRACKER); this.addListenNodeType(NodeType.JOB_TRACKER); } diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java b/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java index 84af90557..5a7625cbc 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/JobClientManager.java @@ -99,7 +99,7 @@ public JobClientNode getAvailableJobClient(String nodeGroup) { JobClientNode jobClientNode = loadBalance.select(application.getConfig(), list, null); if (jobClientNode != null && (jobClientNode.getChannel() == null || jobClientNode.getChannel().isClosed())) { - ChannelWrapper channel = channelManager.getChannel(jobClientNode.getNodeGroup(), NodeType.CLIENT, jobClientNode.getIdentity()); + ChannelWrapper channel = channelManager.getChannel(jobClientNode.getNodeGroup(), NodeType.JOB_CLIENT, jobClientNode.getIdentity()); if (channel != null) { // 更新channel jobClientNode.setChannel(channel); diff --git a/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java b/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java index f18d061b4..e69838d86 100644 --- a/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java +++ b/job-tracker/src/main/java/com/lts/job/tracker/support/listener/JobNodeChangeListener.java @@ -28,7 +28,7 @@ public void addNodes(List nodes) { for (Node node : nodes) { if (node.getNodeType().equals(NodeType.TASK_TRACKER)) { application.getTaskTrackerManager().addNode(node); - } else if (node.getNodeType().equals(NodeType.CLIENT)) { + } else if (node.getNodeType().equals(NodeType.JOB_CLIENT)) { application.getJobClientManager().addNode(node); } } @@ -43,7 +43,7 @@ public void removeNodes(List nodes) { if (node.getNodeType().equals(NodeType.TASK_TRACKER)) { application.getTaskTrackerManager().removeNode(node); application.getDeadJobChecker().fixedDeadLock(node); - } else if (node.getNodeType().equals(NodeType.CLIENT)) { + } else if (node.getNodeType().equals(NodeType.JOB_CLIENT)) { application.getJobClientManager().removeNode(node); } }