Skip to content

Commit 506dcfb

Browse files
swsw
committed
Be more strict about converting float to double
1 parent b6b268d commit 506dcfb

File tree

7 files changed

+71
-63
lines changed

7 files changed

+71
-63
lines changed

‎CMakeLists.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,17 +120,18 @@ if (LLAMA_ALL_WARNINGS)
120120
-Wall
121121
-Wextra
122122
-Wpedantic
123-
-Wshadow
124123
-Wcast-qual
124+
-Wdouble-promotion
125+
-Wshadow
125126
-Wstrict-prototypes
126127
-Wpointer-arith
127-
-Wno-unused-function
128128
)
129129
set(cxx_flags
130130
-Wall
131131
-Wextra
132132
-Wpedantic
133133
-Wcast-qual
134+
-Wdouble-promotion
134135
)
135136
else()
136137
# todo : msvc

‎Makefile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@ endif
3131
#
3232

3333
# keep standard at C11 and C++11
34-
CFLAGS = -I. -O3 -DNDEBUG -std=c11 -fPIC
35-
CXXFLAGS = -I. -I./examples -O3 -DNDEBUG -std=c++11 -fPIC
34+
CFLAGS = -I. -O3 -DNDEBUG -std=c11 -fPIC \
35+
-Wall -Wextra -Wpedantic -Wcast-qual -Wdouble-promotion -Wshadow -Wstrict-prototypes -Wpointer-arith
36+
CXXFLAGS = -I. -I./examples -O3 -DNDEBUG -std=c++11 -fPIC \
37+
-Wall -Wextra -Wpedantic -Wcast-qual -Wdouble-promotion
3638
LDFLAGS =
3739

3840
# OS specific

‎ggml.c

Lines changed: 44 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1270,12 +1270,12 @@ inline static void ggml_vec_dot_f32(const int n, float * restrict s, const float
12701270

12711271
// leftovers
12721272
for (int i = np; i < n; ++i) {
1273-
sumf += x[i]*y[i];
1273+
sumf += (ggml_float)(x[i]*y[i]);
12741274
}
12751275
#else
12761276
// scalar
12771277
for (int i = 0; i < n; ++i) {
1278-
sumf += x[i]*y[i];
1278+
sumf += (ggml_float)(x[i]*y[i]);
12791279
}
12801280
#endif
12811281

@@ -1348,11 +1348,11 @@ inline static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t
13481348

13491349
// leftovers
13501350
for (int i = np; i < n; ++i) {
1351-
sumf += GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]);
1351+
sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
13521352
}
13531353
#else
13541354
for (int i = 0; i < n; ++i) {
1355-
sumf += GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]);
1355+
sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
13561356
}
13571357
#endif
13581358

@@ -1845,13 +1845,13 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * re
18451845
// leftovers
18461846
for (int i = np; i < n; ++i) {
18471847
for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
1848-
sumf[j] += GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]);
1848+
sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
18491849
}
18501850
}
18511851
#else
18521852
for (int i = 0; i < n; ++i) {
18531853
for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
1854-
sumf[j] += GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]);
1854+
sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
18551855
}
18561856
}
18571857
#endif
@@ -2091,11 +2091,11 @@ inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) {
20912091
inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }
20922092
inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }
20932093

2094-
static const ggml_float GELU_COEF_A = 0.044715;
2095-
static const ggml_float SQRT_2_OVER_PI = 0.79788456080286535587989211986876;
2094+
static const float GELU_COEF_A = 0.044715;
2095+
static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876;
20962096

20972097
inline static float ggml_gelu_f32(float x) {
2098-
return 0.5*x*(1.0 + tanh(SQRT_2_OVER_PI*x*(1.0 + GELU_COEF_A*x*x)));
2098+
return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
20992099
}
21002100

21012101
inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
@@ -2124,7 +2124,7 @@ inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
21242124

21252125
// Sigmoid Linear Unit (SiLU) function
21262126
inline static float ggml_silu_f32(float x) {
2127-
return x/(1.0 + exp(-x));
2127+
return x/(1.0f + expf(-x));
21282128
}
21292129

21302130
inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
@@ -2155,7 +2155,7 @@ inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
21552155
#ifndef GGML_USE_ACCELERATE
21562156
ggml_float sum = 0.0;
21572157
for (int i = 0; i < n; ++i) {
2158-
sum += x[i];
2158+
sum += (ggml_float)x[i];
21592159
}
21602160
*s = sum;
21612161
#else
@@ -2165,7 +2165,7 @@ inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
21652165

21662166
inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
21672167
#ifndef GGML_USE_ACCELERATE
2168-
ggml_float max = -INFINITY;
2168+
float max = -INFINITY;
21692169
for (int i = 0; i < n; ++i) {
21702170
max = MAX(max, x[i]);
21712171
}
@@ -2175,7 +2175,10 @@ inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
21752175
#endif
21762176
}
21772177

2178-
inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) { ggml_vec_norm_f32(n, s, x); *s = 1./(*s); }
2178+
inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {
2179+
ggml_vec_norm_f32(n, s, x);
2180+
*s = 1.f/(*s);
2181+
}
21792182

21802183
//
21812184
// logging
@@ -5569,31 +5572,32 @@ static void ggml_compute_forward_norm_f32(
55695572
const size_t nb2 = dst->nb[2];
55705573
const size_t nb3 = dst->nb[3];
55715574

5572-
const ggml_float eps = 1e-5f; // TODO: make this a parameter
5575+
const float eps = 1e-5f; // TODO: make this a parameter
55735576

55745577
// TODO: optimize
55755578
for (int i03 = 0; i03 < ne03; i03++) {
55765579
for (int i02 = 0; i02 < ne02; i02++) {
55775580
for (int i01 = ith; i01 < ne01; i01 += nth) {
55785581
const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
55795582

5580-
ggml_float mean = 0.0;
5583+
ggml_float sum = 0.0;
55815584
for (int i00 = 0; i00 < ne00; i00++) {
5582-
mean += x[i00];
5585+
sum += (ggml_float)x[i00];
55835586
}
55845587

5585-
mean /= ne00;
5588+
float mean = sum/ne00;
55865589

55875590
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
55885591

55895592
ggml_float sum2 = 0.0;
55905593
for (int i00 = 0; i00 < ne00; i00++) {
5591-
ggml_float v = x[i00] - mean;
5594+
float v = x[i00] - mean;
55925595
y[i00] = v;
5593-
sum2 += v*v;
5596+
sum2 += (ggml_float)(v*v);
55945597
}
55955598

5596-
const float scale = 1.0/sqrt(sum2/ne00 + eps);
5599+
float variance = sum2/ne00;
5600+
const float scale = 1.0f/sqrtf(variance + eps);
55975601

55985602
ggml_vec_scale_f32(ne00, y, scale);
55995603
}
@@ -5651,20 +5655,20 @@ static void ggml_compute_forward_rms_norm_f32(
56515655
const size_t nb2 = dst->nb[2];
56525656
const size_t nb3 = dst->nb[3];
56535657

5654-
const ggml_float eps = 1e-6f; // TODO: make this a parameter
5658+
const float eps = 1e-6f; // TODO: make this a parameter
56555659

56565660
// TODO: optimize
56575661
for (int i03 = 0; i03 < ne03; i03++) {
56585662
for (int i02 = 0; i02 < ne02; i02++) {
56595663
for (int i01 = ith; i01 < ne01; i01 += nth) {
56605664
const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
56615665

5662-
ggml_float mean = 0.0;
5666+
ggml_float sum = 0.0;
56635667
for (int i00 = 0; i00 < ne00; i00++) {
5664-
mean += x[i00] * x[i00];
5668+
sum += (ggml_float)(x[i00] * x[i00]);
56655669
}
56665670

5667-
mean /= ne00;
5671+
float mean = sum/ne00;
56685672

56695673
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
56705674

@@ -5673,7 +5677,7 @@ static void ggml_compute_forward_rms_norm_f32(
56735677
// y[i00] = x[i00];
56745678
// }
56755679

5676-
const float scale = 1.0/sqrt(mean + eps);
5680+
const float scale = 1.0f/sqrtf(mean + eps);
56775681

56785682
ggml_vec_scale_f32(ne00, y, scale);
56795683
}
@@ -7328,12 +7332,12 @@ static void ggml_compute_forward_soft_max_f32(
73287332
ggml_fp16_t s = GGML_FP32_TO_FP16(p[i] - max);
73297333
memcpy(&scvt, &s, sizeof(scvt));
73307334
const float val = GGML_FP16_TO_FP32(table_exp_f16[scvt]);
7331-
sum += val;
7335+
sum += (ggml_float)val;
73327336
p[i] = val;
73337337
}
73347338
}
73357339

7336-
assert(sum > 0.0f);
7340+
assert(sum > 0.0);
73377341

73387342
sum = 1.0/sum;
73397343
ggml_vec_scale_f32(nc, p, sum);
@@ -8197,7 +8201,7 @@ static void ggml_compute_forward_flash_attn_f32(
81978201
float max = -INFINITY;
81988202
ggml_vec_max_f32(M, &max, S);
81998203

8200-
float sum = 0.0f;
8204+
ggml_float sum = 0.0;
82018205
{
82028206
#ifdef GGML_SOFT_MAX_ACCELERATE
82038207
max = -max;
@@ -8218,7 +8222,7 @@ static void ggml_compute_forward_flash_attn_f32(
82188222
ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);
82198223
memcpy(&scvt[j], &s, sizeof(uint16_t));
82208224
const float val = GGML_FP16_TO_FP32(table_exp_f16[scvt[j]]);
8221-
sump[j] += val;
8225+
sump[j] += (ggml_float)val;
82228226
SS[j] = val;
82238227
}
82248228
}
@@ -8230,7 +8234,7 @@ static void ggml_compute_forward_flash_attn_f32(
82308234
#endif
82318235
}
82328236

8233-
assert(sum > 0.0f);
8237+
assert(sum > 0.0);
82348238

82358239
sum = 1.0/sum;
82368240
ggml_vec_scale_f32(M, S, sum);
@@ -8423,7 +8427,7 @@ static void ggml_compute_forward_flash_attn_f16(
84238427
float max = -INFINITY;
84248428
ggml_vec_max_f32(M, &max, S);
84258429

8426-
float sum = 0.0f;
8430+
ggml_float sum = 0.0;
84278431
{
84288432
#ifdef GGML_SOFT_MAX_ACCELERATE
84298433
max = -max;
@@ -8444,7 +8448,7 @@ static void ggml_compute_forward_flash_attn_f16(
84448448
ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);
84458449
memcpy(&scvt[j], &s, sizeof(uint16_t));
84468450
const float val = GGML_FP16_TO_FP32(table_exp_f16[scvt[j]]);
8447-
sump[j] += val;
8451+
sump[j] += (ggml_float)val;
84488452
SS[j] = val;
84498453
}
84508454
}
@@ -8456,7 +8460,7 @@ static void ggml_compute_forward_flash_attn_f16(
84568460
#endif
84578461
}
84588462

8459-
assert(sum > 0.0f);
8463+
assert(sum > 0.0);
84608464

84618465
sum = 1.0/sum;
84628466
ggml_vec_scale_f32(M, S, sum);
@@ -9987,7 +9991,7 @@ label=\"%d [%d, %d] | <x>%s",
99879991
fprintf(fp, " \"%p\" [ \
99889992
style = filled; fillcolor = %s; shape = record; \
99899993
label=\"<x>%.1e\"; ]\n",
9990-
(void *) node, color, ggml_get_f32_1d(node, 0));
9994+
(void *) node, color, (double)ggml_get_f32_1d(node, 0));
99919995
} else {
99929996
fprintf(fp, " \"%p\" [ \
99939997
style = filled; fillcolor = %s; shape = record; \
@@ -10225,7 +10229,7 @@ static enum ggml_opt_result ggml_opt_adam(
1022510229
if (params.past <= t) {
1022610230
const float rate = (pf[t%params.past] - fx)/fx;
1022710231

10228-
if (fabs(rate) < params.delta) {
10232+
if (fabsf(rate) < params.delta) {
1022910233
return GGML_OPT_OK;
1023010234
}
1023110235
}
@@ -10304,7 +10308,7 @@ static enum ggml_opt_result linesearch_backtracking(
1030410308
const float dec = 0.5f;
1030510309
const float inc = 2.1f;
1030610310

10307-
if (*step <= 0.) {
10311+
if (*step <= 0.f) {
1030810312
return GGML_LINESEARCH_INVALID_PARAMETERS;
1030910313
}
1031010314

@@ -10392,7 +10396,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(
1039210396
struct ggml_cgraph * gb) {
1039310397
if (params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE ||
1039410398
params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE) {
10395-
if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1. <= params.lbfgs.wolfe) {
10399+
if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1.f <= params.lbfgs.wolfe) {
1039610400
return GGML_OPT_INVALID_WOLFE;
1039710401
}
1039810402
}
@@ -10513,8 +10517,8 @@ static enum ggml_opt_result ggml_opt_lbfgs(
1051310517

1051410518
GGML_PRINT_DEBUG("f = %10.6f\n", ggml_get_f32_1d(f, 0));
1051510519

10516-
if (xnorm < 1.0) {
10517-
xnorm = 1.0;
10520+
if (xnorm < 1.0f) {
10521+
xnorm = 1.0f;
1051810522
}
1051910523
if (gnorm/xnorm <= params.lbfgs.eps) {
1052010524
// converged
@@ -10527,7 +10531,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(
1052710531
if (params.past <= k) {
1052810532
const float rate = (pf[k%params.past] - fx)/fx;
1052910533

10530-
if (fabs(rate) < params.delta) {
10534+
if (fabsf(rate) < params.delta) {
1053110535
return GGML_OPT_OK;
1053210536
}
1053310537
}

0 commit comments

Comments
 (0)