GODEL 代码阅读 [1]

这么看代码可能效率很低 但是我还是想先仔细逐行分析一下完整的project(半记忆背诵的性质) 然后尝试复刻做一个完整的自己的东西 也是因为这个代码量比较小才会这么做

-parser Finetune argument

parser.add_argument(

1. dataset_name, dataset_config
2. train, test, validation file(csv, json)
3. max_source_length(max input sequence length after tokenization)
4. source_prefix(T5)
5. number of processes
6. max_target_length, max_validation_target
7. num_beams()
8. model, config, tokenizer name(pretrained config表示?)
9. text, summary_column(datasets中 包含full text和summary的column in str)
10. slow tokenizer option(不用huggingface tokenizer)
11. batch size, learning rate, weight decay, #epoch(与max_train_steps区别)
12. gradient_accumulation_steps(#updates steps to accumulate before performing a backward/update pass)
13. out_put dir(for store trained model), seed, model type(for training from scratch)
14. max_length, pad_to_max_length(与max_source_length区别?), ignore_pad_token_for_loss
15. logging, save steps, checkpoint_save
16. max_grad_norm

)

args = parser.parse_args()

-Main

from accelerate import Accelerator

#Manage device placement automatically
accelerator = Accelerator()

logger = logging.getLogger(__name__)

logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO,
    )
logger.info(accelerator.state)

使用wandb 去initializes Weights & Biases (WandB) for tracking experiment metrics

Get dataset

1.load_dataset(args.dataset_name, args.dataset_config_name)

with dataset_name from public dataset in huggingface hub

1. load_dataset(extension, data_files=data_files)

data_files = dict[”train”, “validation”…]

extension = csv ^ json

Load config

config =

AutoConfig.from_pretrained(config_name ^ from model_name_path)

^ CONFIG_MAPPING(model_type) (Create new config)

Load tokenizer

tokenizer =

AutoTokenizer.from_pretrained(tokenizer_name^ from model_name_path)

Load model

model =

AutoModelForSeq2SeqLM.from_pretrained(model_name_path, config)

^ AutoModelForSeq2SeqLM.from_config(config)

Add Padding token “PAD” to tokenizer →

model.resize_token_embedding(len(tokenizer))

Set up model.config.decoder_start_token_id

Dataset mapping method

inputs.append(context + (knowledgeBase))

model_input = tokenizer(inputs, max_length, padding, truncation = True)

labels = tokenizer(responses, max_target_length, padding, truncation = True)

model_inputs[“labels”] = labels[“labels”]

若是 max_length padding(ignore padding in the loss)

if padding == "max_length" and args.ignore_pad_token_for_loss:
            labels["labels"] = [
                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"]
            ]

Convert raw dataset and initialize data loader

#用到了multiprocessing

lm_datasets = raw_dataset.map(mapping_func, remove_column, num_proc…)

train_dataset = lm_datasets[“train”]…

label_pad_token_id = -100 if args.ignore_pad_token_for_loss else tokenizer.pad_token_id

Build Collator for data loader

#这里面也要用到pad_token_id
label_pad_token_id = -100 if args.ignore_pad_token_for_loss else tokenizer.pad_token_id
data_collator = DataCollatorForSeq2Seq(
    tokenizer,
    model=model,
    label_pad_token_id=label_pad_token_id,
    pad_to_multiple_of=8 if accelerator.use_fp16 else None,
)

train_dataloader = DataLoader(
        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size
    )

Optimizer

# Optimizer
# Split weights in two groups, one with weight decay and the other not.
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
    {
        "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
        "weight_decay": args.weight_decay,
    },
    {
        "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
        "weight_decay": 0.0,
    },
]
#区分weight decay parameters(penalizing large weights, but doesn't disturb layer norm)
optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate)

Scheduling

num_update_per_epoch = ceil(len(dataloader)(# of batch per epoch) / gradient_accumulate_step)

max_train_steps(schedular参数) = epochs * num_update_per_epoch

lr_scheduler = get_scheduler(
    name=args.lr_scheduler_type,
    optimizer=optimizer,
    num_warmup_steps=args.num_warmup_steps,
    num_training_steps=args.max_train_steps,
)                                                   

Training

for epoch in range(num_epoch):

for step, batch in enumerate(dataloader):

global_steps += 1

outputs = model(**batch)

loss = outputs.loss / gradient_accumulation_steps

accelerator.backward(loss)

if step % args.gradient_accumulate_step == 0 or step == len(dataloader) - 1:

optimizer.step()

scheduler.step()

optimizer.zero_grad()

completed_step += 1

#Total update count

if completed_step ≥ max_train_steps:

break

#模型保存

if output_dir and global_steps % steps_to_save == 0:

accelerator.wait_for_everyone()

unwrapped_model = accelerator.unwrap_model(model)

unwrapped_model.save_pretrained(output_dir)

tokenizer.save_pretrained(output_dir)

torch.save(arts, output_dir + training_args.bin)