'SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'DecayedAdagrad', 'Ftrl',

'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', 'AdamOptimizer',

'AdamaxOptimizer', 'DecayedAdagradOptimizer', 'RMSPropOptimizer',

'FtrlOptimizer', 'Adadelta', 'ModelAverage', 'LarsMomentum',

'LarsMomentumOptimizer', 'DGCMomentumOptimizer', 'LambOptimizer',

'ExponentialMovingAverage'

"""Optimizer Base class.

def __init__(self, learning_rate, regularization=None, name=None):

if framework.in_dygraph_mode():

if not isinstance(learning_rate, float) and \

not isinstance(learning_rate, LearningRateDecay):

raise TypeError(

"learning rate should be float or LearningRateDecay, got %s here"

% type(learning_rate))

else:

if not isinstance(learning_rate, float) and \

not isinstance(learning_rate, framework.Variable):

raise TypeError(

"learning rate should be float or Variable, got %s here" %

type(learning_rate))

self._name = name

self.regularization = regularization

self._learning_rate = learning_rate

# the learning rate type should be inferenced from loss

self._dtype = None

# each program should have a independent learning rate

# program -> Variable(learning_rate)

self._learning_rate_map = dict()

if isinstance(self._learning_rate, framework.Variable):

self._learning_rate_map[framework.default_main_program(

)] = self._learning_rate

# Dictionary of accumulators. Some optimizer subclasses need to

# allocate and manage extra variables associated with the parameters

# to train. These variables are called accumulators.

# {accum_name : { paramter_name : accumulator_for_parameter, ...}, ...}

self._accumulators = defaultdict(lambda: dict())

self.helper = None

self._opti_name_list = []

def get_opti_var_name_list(self):

return self._opti_name_list

def _create_global_learning_rate(self):

if imperative_base.enabled():

# create learning rate Variable

if isinstance(self._learning_rate, float):

lr = self._global_learning_rate()

if isinstance(lr, framework.Variable):

return

else:

self._learning_rate_map[framework.default_main_program(

)] = layers.create_global_var(

name=unique_name.generate("learning_rate"),

shape=[1],

value=float(self._learning_rate),

dtype='float32' if self._dtype is None else self._dtype,

persistable=True)

# get learning rate Variable from LearningRateDecay

elif isinstance(self._learning_rate, LearningRateDecay):

self._learning_rate_map[framework.default_main_program(

)] = self._learning_rate()

else:

raise TypeError(

"optimizer's learning rate must be float or LearningRateDecay"

)

else:

lr = self._global_learning_rate()

if isinstance(lr, framework.Variable):

return

else:

if not isinstance(self._learning_rate, float):

raise TypeError(

"learning rate variable is create outside optimizer,"

"can not create new learning rate variable for new program"

)

# create learning rate in the current main program

self._learning_rate_map[framework.default_main_program(

)] = layers.create_global_var(

name=unique_name.generate("learning_rate"),

shape=[1],

value=float(self._learning_rate),

dtype='float32' if self._dtype is None else self._dtype,

persistable=True)

def _global_learning_rate(self, program=None):

"""

if program is None:

program = framework.default_main_program()

return self._learning_rate_map.get(program, None)

def _append_optimize_op(self, block, param_and_grad):

""" append optimize operator to block and return all the added optimize_op

raise NotImplementedError()

def _create_param_lr(self, param_and_grad):

# create learning rate variable for every parameter

param = param_and_grad[0]

param_lr = param.optimize_attr['learning_rate']

if type(param_lr) == Variable:

return param_lr

else:

if param_lr == 1.0:

return self._global_learning_rate()

else:

with default_main_program()._lr_schedule_guard(

is_with_opt=True), framework.name_scope(

'scale_with_param_lr'):

return self._global_learning_rate() * param_lr

def _create_accumulators(self, block, parameters):

"""Create all accumulators needed by the parameters

pass

def _finish_update(self, block, parameters_and_grads):

"""Finish any custom updates needed

pass

def _add_accumulator(self,

name,

param,

dtype=None,

fill_value=0.0,

shape=None):

"""Utility function to add an accumulator for a parameter

if self._name is not None:

name = self._name + "_" + name

if (name in self._accumulators and

param.name in self._accumulators[name]):

if framework.in_dygraph_mode():

return self._accumulators[name][param.name]

raise Exception("Accumulator {} already exists for parameter {}".

format(name, param.name))

if shape == None:

shape = param.shape

assert isinstance(self.helper, LayerHelper)

var_name = param.name + "_" + name

var_name = unique_name.generate(var_name)

self._opti_name_list.append(var_name)

var = self.helper.create_global_variable(

name=var_name,

persistable=True,

dtype=dtype or param.dtype,

type=param.type,

shape=shape)

self.helper.set_variable_initializer(

var, initializer=Constant(value=float(fill_value)))

self._accumulators[name][param.name] = var

return var

def _get_accumulator(self, name, param):

"""Utility function to fetch an accumulator for a parameter

if self._name is not None:

name = self._name + "_" + name

if (name not in self._accumulators or

param.name not in self._accumulators[name]):

raise Exception("Accumulator {} does not exist for parameter {}".

format(name, param.name))

return self._accumulators[name][param.name]

def _create_optimization_pass(self, parameters_and_grads):

"""Add optimization operators to update gradients to variables.

# This is a default implementation of create_optimization_pass that

# can be shared by most optimizers. This implementation assumes that

# the subclass will implement the _append_optimize_op method and the

# _initialize_tensors method. The subclass can extend the

# _create_accumulators method if it needs to create accumulators

# for parameters and extend _finish_update method to add custom ops.

# Allways called under program_guard use global block as loss block

global_block = framework.default_main_program().global_block()

start = len(global_block.ops)

self.helper = LayerHelper(self.__class__.__name__)

self._create_accumulators(global_block,

[p[0] for p in parameters_and_grads])

self._create_global_learning_rate()

optimize_ops = []

if framework.in_dygraph_mode():

for param_and_grad in parameters_and_grads:

if param_and_grad[1] is None:

continue

with param_and_grad[0].block.program._optimized_guard(

param_and_grad):

if param_and_grad[0].trainable is True:

optimize_op = self._append_optimize_op(global_block,

param_and_grad)

optimize_ops.append(optimize_op)

else:

for param_and_grad in parameters_and_grads:

if param_and_grad[1] is None:

continue

with param_and_grad[0].block.program._optimized_guard(

param_and_grad), name_scope("optimizer"):

if param_and_grad[0].trainable is True:

optimize_op = self._append_optimize_op(global_block,

param_and_grad)

optimize_ops.append(optimize_op)

# Get custom finish ops for subclasses

# FIXME: Need to fix this once we figure out how to handle dependencies

self._finish_update(global_block, parameters_and_grads)

end = len(global_block.ops)

return global_block._slice_ops(start, end)

def _process_distribute_lookuptable(self, param_grads):

"""

program = framework.default_main_program()

global_block = framework.default_main_program().global_block()

table_name = find_distributed_lookup_table(program)

table_param = None

table_grad = None

new_param_grads = []

for p, g in param_grads:

if p.name == table_name:

if table_param is not None:

raise RuntimeError(

"multi dist table var found, only support one now!")

table_param = p

table_grad = g

else:

new_param_grads.append((p, g))

sgd_op = None

if table_param is not None:

param_and_grad = [table_param, table_grad]

with table_param.block.program._optimized_guard(param_and_grad), \

framework.name_scope("optimizer"):

self._create_global_learning_rate()

# create the optimize op

sgd_op = global_block.append_op(

type='sgd',

inputs={

"Param": table_param,

"Grad": table_grad,

"LearningRate": self._create_param_lr(param_and_grad)

},

outputs={"ParamOut": param_and_grad[0]})

return new_param_grads, (table_param, table_grad), sgd_op

def _append_dgc_ops(self, param_and_grad):

pass

def backward(self,

loss,

startup_program=None,

parameter_list=None,

no_grad_set=None,

callbacks=None):

"""

self._dtype = loss.dtype

if framework.in_dygraph_mode():

if parameter_list is not None:

parameters = parameter_list

else:

parameters = framework._dygraph_tracer().all_parameters()

params_grads = []

for param in parameters:

if not param.trainable:

continue

if param._ivar._grad_ivar() is not None:

# create gradient variable

grad_var = Variable(

block=loss.block,

name=param._ivar._grad_name(),

stop_gradient=True,

ivar=param._ivar._grad_ivar())

params_grads.append((param, grad_var))

else:

if callbacks is None:

callbacks = [error_clip_callback]

else:

assert (isinstance(callbacks, list))

program = loss.block.program

with program_guard(program, startup_program):

params_grads = append_backward(loss, parameter_list,

no_grad_set, callbacks)

# Note: since we can't use all_reduce_op now,

# dgc_op should be the last op of one grad.

self._append_dgc_ops(params_grads)

return params_grads

def apply_gradients(self, params_grads):

"""

params_grads = sorted(params_grads, key=lambda x: x[0].name)

params_grads, table_param_and_grad, table_optimize_op = \

self._process_distribute_lookuptable(params_grads)

params_grads = append_gradient_clip_ops(params_grads)

# Add regularization if any

params_grads = append_regularization_ops(params_grads,

self.regularization)

optimize_ops = self._create_optimization_pass(params_grads)

if table_optimize_op is not None:

optimize_ops.append(table_optimize_op)

params_grads.append(table_param_and_grad)

return optimize_ops

def apply_optimize(self, loss, startup_program, params_grads):

"""

if framework.in_dygraph_mode():

with program_guard(framework.default_main_program(),

framework.default_startup_program()):

params_grads = append_regularization_ops(params_grads,

self.regularization)

optimize_ops = self._create_optimization_pass(params_grads)

else:

program = loss.block.program

with program_guard(program, startup_program):

optimize_ops = self.apply_gradients(params_grads)

return optimize_ops

def minimize(self,

loss,

startup_program=None,

parameter_list=None,

no_grad_set=None,

grad_clip=None):

"""

params_grads = self.backward(

loss,

startup_program=startup_program,

parameter_list=parameter_list,

no_grad_set=no_grad_set)

if grad_clip is not None and framework.in_dygraph_mode():

# TODO(hongyu): FIX later, this is only for dygraph, should be work for static mode

params_grads = grad_clip(params_grads)

optimize_ops = self.apply_optimize(

loss, startup_program=startup_program, params_grads=params_grads)

if framework.in_dygraph_mode():

framework._dygraph_tracer()._clear_ops()

"""

def __init__(self, learning_rate, regularization=None, name=None):

assert learning_rate is not None

super(SGDOptimizer, self).__init__(

learning_rate=learning_rate,

regularization=regularization,

name=name)

self.type = "sgd"

def _append_optimize_op(self, block, param_and_grad):

assert isinstance(block, framework.Block)

# create the optimize op

sgd_op = block.append_op(

type=self.type,

inputs={

"Param": param_and_grad[0],

"Grad": param_and_grad[1],

"LearningRate": self._create_param_lr(param_and_grad)

},

outputs={"ParamOut": param_and_grad[0]},

stop_gradient=True)

"""

_velocity_acc_str = "velocity"

def __init__(self,

learning_rate,

momentum,

use_nesterov=False,

regularization=None,

name=None):

assert learning_rate is not None

assert momentum is not None

super(MomentumOptimizer, self).__init__(

learning_rate=learning_rate,

regularization=regularization,

name=name)

self.type = "momentum"

self._momentum = momentum

self._use_nesterov = bool(use_nesterov)

def _create_accumulators(self, block, parameters):

assert isinstance(block, framework.Block)

for p in parameters:

self._add_accumulator(self._velocity_acc_str, p)

def _append_optimize_op(self, block, param_and_grad):

assert isinstance(block, framework.Block)

velocity_acc = self._get_accumulator(self._velocity_acc_str,

param_and_grad[0])

# create the momentum optimize op

momentum_op = block.append_op(

type=self.type,

inputs={

"Param": param_and_grad[0],

"Grad": param_and_grad[1],

"Velocity": velocity_acc,

"LearningRate": self._create_param_lr(param_and_grad)

},

outputs={

"ParamOut": param_and_grad[0],

"VelocityOut": velocity_acc

},

attrs={"mu": self._momentum,

"use_nesterov": self._use_nesterov},

stop_gradient=True)

"""

def __init__(self,

learning_rate,

momentum,

rampup_begin_step,

rampup_step=1,

sparsity=[0.999],

use_nesterov=False,

local_grad_clip_norm=None,

num_trainers=None,

regularization=None,

name=None):

self._sparsity = sparsity

self._rampup_step = rampup_step

self._rampup_step_var = None

self._rampup_begin_step = rampup_begin_step

self._rampup_begin_step_var = None

self._global_step_var = None

self._local_grad_clip_norm = None

self._clip_norm = None

if local_grad_clip_norm is not None:

assert isinstance(num_trainers, int)

assert isinstance(local_grad_clip_norm, float)

assert num_trainers > 0

self._local_grad_clip_norm = local_grad_clip_norm

self._num_trainers = num_trainers

self._clip_norm = local_grad_clip_norm / (num_trainers *

num_trainers)

super(DGCMomentumOptimizer, self).__init__(

learning_rate, momentum, use_nesterov, regularization, name)

core.init_dgc()

def _add_auto_increment_var(self, counter_name, begin, step=1):

helper = LayerHelper('global_step_counter')

counter, is_new_var = helper.create_or_get_global_variable(

name=counter_name, dtype='float32', shape=[1], persistable=True)

if is_new_var:

helper.set_variable_initializer(

counter,

initializer=Constant(

value=float(begin - 1), force_cpu=True))

helper.main_program.global_block()._prepend_op(

type='increment',

inputs={'X': [counter]},

outputs={'Out': [counter]},

attrs={'step': float(step)},

stop_gradient=True)

counter.stop_gradient = True

return counter

def _append_dgc_ops(self, param_and_grads):

start_program = default_startup_program()

main_program = default_main_program()

main_program._enable_dgc = True

# step counter

self._global_step_var = self._add_auto_increment_var(

counter_name=core.dgc.kDGCCounterName(), begin=0)

# rampup begin step var for all_reduce_op_handle

self._rampup_begin_step_var = tensor.create_global_var(

shape=[1],

dtype=core.VarDesc.VarType.FP32,

persistable=True,

name=core.dgc.kDGCRampUpBeginStepName(),

value=self._rampup_begin_step * 1.0,

force_cpu=True)

for param_var, grad_var in param_and_grads:

var_numel = abs(reduce(lambda x, y: x * y, param_var.shape))

if var_numel < 16384 or \

param_var.type == core.VarDesc.VarType.SELECTED_ROWS or \

grad_var.type == core.VarDesc.VarType.SELECTED_ROWS or \

param_var.dtype != core.VarDesc.VarType.FP32 :

continue

u_var = tensor.create_global_var(

shape=param_var.shape,

dtype=param_var.dtype,

persistable=True,

name=param_var.name + core.dgc.kDGCUName(),

value=0.0)

v_var = tensor.create_global_var(

shape=param_var.shape,

dtype=param_var.dtype,

persistable=True,

name=param_var.name + core.dgc.kDGCVName(),

value=0.0)

k_var = tensor.create_global_var(

shape=[1],

dtype=param_var.dtype,

persistable=True,

name=param_var.name + core.dgc.kDGCKName(),

value=0.0,

force_cpu=True)

encoded_var = tensor.create_global_var(

shape=[1],

dtype=param_var.dtype,

persistable=True,

name=param_var.name + core.dgc.kDGCEncodedName(),

value=0.0,

force_cpu=False)

# del back oprolevarname

op_maker = core.op_proto_and_checker_maker

backward = core.op_proto_and_checker_maker.OpRole.Backward

for op in main_program.global_block().ops:

if not self._is_the_backward_op(op):

continue

var_attr = op.all_attrs()[op_maker.kOpRoleVarAttrName()]

if param_var.name not in var_attr:

continue

var_attr.remove(param_var.name)

var_attr.remove(grad_var.name)

if len(var_attr) > 1:

op._set_attr(op_maker.kOpRoleVarAttrName(), var_attr)

else:

op._remove_attr(op_maker.kOpRoleVarAttrName())

clip_var = grad_var

if self._local_grad_clip_norm is not None:

clip_var = self._append_clip_norm(grad_var, self._clip_norm)

self._dgc_op(param_var, clip_var, grad_var, u_var, v_var, k_var,

encoded_var)

def _is_the_backward_op(self, op):

op_maker = core.op_proto_and_checker_maker

backward = core.op_proto_and_checker_maker.OpRole.Backward

if op_maker.kOpRoleVarAttrName() in op.attr_names and \

int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward):

return True

return False

def _clip_by_norm(self, x, max_norm, name=None):

args = {'x': x, 'max_norm': max_norm, 'name': name}

helper = LayerHelper("dgc_clip_by_norm_op", **args)

if name is None:

name = unique_name.generate_with_ignorable_key(".".join(

[helper.name, 'tmp']))

out = helper.create_variable(

type=x.type, name=name, dtype=x.dtype, persistable=False)

helper.append_op(

type="dgc_clip_by_norm",

inputs={"X": x,

"current_step": self._global_step_var},

attrs={

"max_norm": max_norm,

"rampup_begin_step": float(self._rampup_begin_step)

},

outputs={"Out": out})

return out

def _append_clip_norm(self, grad_var, clip_norm):

with grad_var.block.program._backward_role_guard():

return self._clip_by_norm(

x=grad_var, max_norm=clip_norm, name=grad_var.name)

def _dgc_op(self, param_var, clip_var, grad_var, u_var, v_var, k_var,

encoded_var):

block = framework.default_main_program().global_block()

op_maker = core.op_proto_and_checker_maker

dgc_op = block.append_op(

type="dgc",

inputs={

"U": u_var,

"V": v_var,

"Grad": clip_var,

"current_step": self._global_step_var

},

outputs={

"U_out": u_var,

"V_out": v_var,

"EncodeGrad": encoded_var,

"k": k_var,

"Grad_out": grad_var

},

attrs={

"m": self._momentum,

"sparsity": self._sparsity,

"use_nesterov": self._use_nesterov,

"rampup_begin_step": float(self._rampup_begin_step),

"rampup_step": float(self._rampup_step)

},

stop_gradient=True)

backward = op_maker.OpRole.Backward

dgc_op._set_attr(op_maker.kOpRoleAttrName(), backward)

dgc_op._set_attr(op_maker.kOpRoleVarAttrName(),

"""

_velocity_acc_str = "velocity"

def __init__(self,

learning_rate,

momentum,

lars_coeff=0.001,

lars_weight_decay=0.0005,

regularization=None,

name=None):

assert learning_rate is not None

assert momentum is not None

super(LarsMomentumOptimizer, self).__init__(

learning_rate=learning_rate,

regularization=regularization,

name=name)