override def compute(split: Partition, context: TaskContext): Iterator[T] = { val file = new Path(checkpointPath, ReliableCheckpointRDD.checkpointFileName(split.index)) ReliableCheckpointRDD.readCheckpointFile(file, broadcastedConf, context) } 1. 2. 3. 4. 5. 第三步:readCheckpointFile方法 def ...
compute(split, context) } } 当调用rdd.iterator()去计算该 rdd 的 partition 的时候,会调用computeOrReadCheckpoint(split: Partition)去查看该 rdd 是否被 checkpoint 过了,如果是,就调用该 rdd 的 parent rdd 的 iterator() 也就是 CheckpointRDD.iterator(),否则直接调用该RDD的compute, 那么我们就跟进Checkp...
override def compute(split: Partition, context: TaskContext): Iterator[T] = { val file = new Path(checkpointPath, ReliableCheckpointRDD.checkpointFileName(split.index)) ReliableCheckpointRDD.readCheckpointFile(file, broadcastedConf, context) } /** * Read the content of the specified checkpoint file...
val file = new Path(checkpointPath, ReliableCheckpointRDD.checkpointFileName(split.index)) ReliableCheckpointRDD.readCheckpointFile(file, broadcastedConf, context) } 1. 2. 3. 4. 5. 6. 7. 这里就两行代码,意思是从Path上读取我们的CheckPoint数据,看一下readCheckpointFile /** * Read the content o...
override def(split: Partition, context: TaskContext): Iterator[T] = { val file = new Path(checkpointPath, ReliableCheckpointRDD.checkpointFileName(split.index)) ReliableCheckpointRDD.readCheckpointFile(file, broadcastedConf, context) } /**
compute(split, context) } } ReliableCheckpointRDD.compute overridedefcompute(split:Partition, context:TaskContext):Iterator[T] = {valfile =newPath(checkpointPath,ReliableCheckpointRDD.checkpointFileName(split.index))ReliableCheckpointRDD.readCheckpointFile(file, broadcastedConf, context) ...
*/private[spark] def computeOrReadCheckpoint(split: Partition, context: TaskContext): Iterator[T] = {if(isCheckpointed) { firstParent[T].iterator(split, context) }else{ compute(split, context) } } RDDCheckpointData /** * Enumeration to manage state transitions of an RDD through checkpointin...
DTS_E_SPLITEXPRESSIONNOTBOOLEAN DTS_E_SQLCEDESTDATATYPENOTSUPPORTED DTS_E_SQLCEDESTSTATIC_FAILEDTOINSERT DTS_E_SQLCEDESTSTATIC_FAILEDTOSETVALUES DTS_E_SQLPERSISTENCEVERSION DTS_E_SQLPROFILERDLL_ONLY_X86 DTS_E_SQLTASK_CANNOTACQUIRECONNMANAGER DTS_E_SQLTASK_CANNOTLOCATECONNMANAGER DTS_E_SQLTASK_CO...
{val path="/bigdatalearnshare/checkpointLocation/binlog-2-kafka/offsets/2"val fs=FileSystem.get(newConfiguration())val lastFile=fs.listStatus(newPath(path)).filterNot(_.getPath.getName.endsWith(".tmp.crc")).map{fileName=>(fileName.getPath.getName.split("/").last.toInt,fileName.get...
Whitespace Ignore whitespace Split Unified examples/mixtral README.md train_mixtral_8x7b_distributed.sh megatron core/transformer/moe router.py legacy/model transformer.py training arguments.py checkpointing.py pretrain_gpt.py tools/checkpoint loader_llama_mistral.py loader_mcore.py lo...