forked from Tensor-Array/Tensor-Array
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtransformer.hh
More file actions
83 lines (71 loc) · 2.77 KB
/
transformer.hh
File metadata and controls
83 lines (71 loc) · 2.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
/*
Copyright 2024 TensorArray-Creators
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include "attention.hh"
#pragma once
#ifdef _WIN32
#ifdef TENSOR_ARRAY_LAYERS_EXPORTS
#define TENSOR_ARRAY_API __declspec(dllexport)
#else
#define TENSOR_ARRAY_API __declspec(dllimport)
#endif
#else
#define TENSOR_ARRAY_API
#endif
namespace tensor_array
{
namespace layers
{
class TENSOR_ARRAY_API TransformerEncoderImpl final :
public TensorCalculateLayerImpl
{
private:
MultiHeadAttention multihead_attn;
Sequential feed_forward;
Normalization
layer_norm_1 = Normalization(std::initializer_list<unsigned char>{1}),
layer_norm_2 = Normalization(std::initializer_list<unsigned char>{1});
public:
TransformerEncoderImpl(unsigned int, unsigned int, unsigned int);
value::Tensor calculate(const value::Tensor&) override;
};
using TransformerEncoder = LayerHolder<TransformerEncoderImpl>;
class TENSOR_ARRAY_API TransformerDecoderImpl final :
public LayerImpl
{
private:
MultiHeadAttention masked_multihead_attn, multihead_attn;
Sequential feed_forward;
Normalization
layer_norm_1 = Normalization(std::initializer_list<unsigned char>{1}),
layer_norm_2 = Normalization(std::initializer_list<unsigned char>{1}),
layer_norm_3 = Normalization(std::initializer_list<unsigned char>{1});
public:
TransformerDecoderImpl(unsigned int, unsigned int, unsigned int);
value::Tensor calculate(const value::Tensor&, const value::Tensor&, const value::Tensor&);
};
using TransformerDecoder = LayerHolder<TransformerDecoderImpl>;
class TransformerImpl final :
public LayerImpl
{
private:
Sequential encoder_blocks;
std::vector<TransformerDecoder> decoder_blocks;
Linear fc;
public:
TransformerImpl(unsigned int, unsigned int, unsigned int, unsigned int);
value::Tensor calculate(const value::Tensor&, const value::Tensor&, const value::Tensor&);
};
using Transformer = LayerHolder<TransformerImpl>;
}
}
#undef TENSOR_ARRAY_API