-
Notifications
You must be signed in to change notification settings - Fork 19
/
sscal_amd64.s
118 lines (91 loc) · 2.01 KB
/
sscal_amd64.s
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
//func Sscal(N int, alpha float32, X []float32, incX int)
TEXT ·Sscal(SB), 7, $0
MOVQ N+0(FP), BP
MOVSS alpha+8(FP), X0
MOVQ X_data+16(FP), SI
MOVQ incX+40(FP), AX
// Check data bounaries
MOVQ BP, CX
DECQ CX
IMULQ AX, CX // CX = incX * (N - 1)
CMPQ CX, X_len+24(FP)
JGE panic
// Setup stride
SALQ $2, AX // AX = sizeof(float32) * incX
// Check that there are 4 or more pairs for SIMD calculations
SUBQ $4, BP
JL rest // There are less than 4 values to process
// Setup four alphas in X0
SHUFPS $0, X0, X0
// Check if incX != 1
CMPQ AX, $4
JNE with_stride
// Fully optimized loop (for incX == 1)
full_simd_loop:
// Load four values and scale
MOVUPS (SI), X2
MULPS X0, X2
// Save scaled values
MOVUPS X2, (SI)
// Update data pointers
ADDQ $16, SI
SUBQ $4, BP
JGE full_simd_loop // There are 4 or more pairs to process
JMP rest
with_stride:
// Setup long stride
MOVQ AX, CX
SALQ $1, CX // CX = 8 * incX
// Partially optimized loop
half_simd_loop:
// Load first two values
MOVSS (SI), X2
MOVSS (SI)(AX*1), X4
// Create a half-vector
UNPCKLPS X4, X2
// Save data pointer
MOVQ SI, DI
// Update data pointer using long stride
ADDQ CX, SI
// Load second two values
MOVSS (SI), X4
MOVSS (SI)(AX*1), X6
// Create a half-vector
UNPCKLPS X6, X4
// Create a full-vector
MOVLHPS X4, X2
// Scale the full-vector
MULPS X0, X2
// Unvectorize and save the result
MOVHLPS X2, X3
MOVSS X2, X4
MOVSS X3, X5
SHUFPS $0xe1, X2, X2
SHUFPS $0xe1, X3, X3
MOVSS X4, (DI)
MOVSS X2, (DI)(AX*1)
MOVSS X5, (SI)
MOVSS X3, (SI)(AX*1)
// Update data pointers using long strides
ADDQ CX, SI
SUBQ $4, BP
JGE half_simd_loop // There are 4 or more pairs to process
rest:
// Undo last SUBQ
ADDQ $4, BP
// Check that are there any value to process
JE end
loop:
// Load from X and save scaled
MOVSS (SI), X2
MULSS X0, X2
MOVSS X2, (SI)
// Update data pointers
ADDQ AX, SI
DECQ BP
JNE loop
end:
RET
panic:
CALL ·panicIndex(SB)
RET