微信公众号搜"智元新知"关注
微信扫一扫可直接关注哦!

org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory的实例源码

项目:hadoop    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,MyResolver.class,DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1,1,true,this.getClass().getName(),true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job,JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerFortest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history",fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move",msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history",fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hadoop    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1,true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job,JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",hfm.movetoDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ",fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hadoop    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1,true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job,JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1,jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1,jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L,10L,null,"default",0L,System.currentTimeMillis() + 1,JobState.SUCCEEDED);

    Assert.assertEquals(1,jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000",jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server",jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:aliyun-oss-hadoop-fs    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:aliyun-oss-hadoop-fs    文件TestJobHistoryParsing.java   
/**
 * Test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();

    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:aliyun-oss-hadoop-fs    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:big-c    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:big-c    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:big-c    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:hadoop-2.6.0-cdh5.4.3    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hadoop-2.6.0-cdh5.4.3    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hadoop-2.6.0-cdh5.4.3    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:hadoop-plus    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate old job history",fileInfo);
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hadoop-plus    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    // should be deleted !
    Assert.assertTrue("file should be deleted ",fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hadoop-plus    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:FlexMap    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:FlexMap    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:FlexMap    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:hops    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hops    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hops    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:hadoop-TCP    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hadoop-TCP    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hadoop-TCP    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:hardfs    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hardfs    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hardfs    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
项目:hadoop-on-lustre2    文件TestJobHistoryParsing.java   
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,hfm.movetoDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
项目:hadoop-on-lustre2    文件TestJobHistoryParsing.java   
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
项目:hadoop-on-lustre2    文件TestJobHistoryParsing.java   
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}

版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。