Skip to content

Commit 5d76b8e

Browse files
swsw
committed
Be more strict about converting float to double
1 parent 502a400 commit 5d76b8e

File tree

7 files changed

+87
-79
lines changed

7 files changed

+87
-79
lines changed

‎CMakeLists.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,17 +120,18 @@ if (LLAMA_ALL_WARNINGS)
120120
-Wall
121121
-Wextra
122122
-Wpedantic
123-
-Wshadow
124123
-Wcast-qual
124+
-Wdouble-promotion
125+
-Wshadow
125126
-Wstrict-prototypes
126127
-Wpointer-arith
127-
-Wno-unused-function
128128
)
129129
set(cxx_flags
130130
-Wall
131131
-Wextra
132132
-Wpedantic
133133
-Wcast-qual
134+
-Wdouble-promotion
134135
)
135136
else()
136137
# todo : msvc

‎Makefile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@ endif
3131
#
3232

3333
# keep standard at C11 and C++11
34-
CFLAGS = -I. -O3 -DNDEBUG -std=c11 -fPIC
35-
CXXFLAGS = -I. -I./examples -O3 -DNDEBUG -std=c++11 -fPIC
34+
CFLAGS = -I. -O3 -DNDEBUG -std=c11 -fPIC \
35+
-Wall -Wextra -Wpedantic -Wcast-qual -Wdouble-promotion -Wshadow -Wstrict-prototypes -Wpointer-arith
36+
CXXFLAGS = -I. -I./examples -O3 -DNDEBUG -std=c++11 -fPIC \
37+
-Wall -Wextra -Wpedantic -Wcast-qual -Wdouble-promotion
3638
LDFLAGS =
3739

3840
# OS specific

‎ggml.c

Lines changed: 54 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -479,8 +479,8 @@ static void quantize_row_q4_0_reference(const float * restrict x, void * restric
479479
const float v0 = x[i*QK + l + 0]*id;
480480
const float v1 = x[i*QK + l + 1]*id;
481481

482-
const uint8_t vi0 = ((int8_t) (round(v0))) + 8;
483-
const uint8_t vi1 = ((int8_t) (round(v1))) + 8;
482+
const uint8_t vi0 = (int8_t)roundf(v0) + 8;
483+
const uint8_t vi1 = (int8_t)roundf(v1) + 8;
484484

485485
assert(vi0 >= 0 && vi0 < 16);
486486
assert(vi1 >= 0 && vi1 < 16);
@@ -747,8 +747,8 @@ void quantize_row_q4_1(const float * restrict x, void * restrict y, int k) {
747747
const float v0 = (x[i*QK + l + 0] - min)*id;
748748
const float v1 = (x[i*QK + l + 1] - min)*id;
749749

750-
const uint8_t vi0 = round(v0);
751-
const uint8_t vi1 = round(v1);
750+
const uint8_t vi0 = roundf(v0);
751+
const uint8_t vi1 = roundf(v1);
752752

753753
assert(vi0 >= 0 && vi0 < 16);
754754
assert(vi1 >= 0 && vi1 < 16);
@@ -1371,7 +1371,7 @@ inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, co
13711371
inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }
13721372

13731373
inline static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y) {
1374-
ggml_float sumf = 0.0;
1374+
float sumf = 0.0f;
13751375

13761376
#ifdef GGML_SIMD
13771377
const int np = (n & ~(GGML_F32_STEP - 1));
@@ -1449,7 +1449,7 @@ static inline __m512 dot_q4_0_oneblock_avx512(
14491449
#endif
14501450

14511451
inline static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y) {
1452-
ggml_float sumf = 0.0;
1452+
float sumf = 0.0f;
14531453

14541454
#if defined(GGML_SIMD)
14551455
const int np = (n & ~(GGML_F16_STEP - 1));
@@ -1934,7 +1934,7 @@ inline static void ggml_vec_dot_q4_1(const int n, float * restrict s, const void
19341934
// compute GGML_VEC_DOT_UNROLL dot products at once
19351935
// xs - x row stride in bytes
19361936
inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {
1937-
ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };
1937+
float sumf[GGML_VEC_DOT_UNROLL] = { 0.0f };
19381938

19391939
ggml_fp16_t * restrict x[GGML_VEC_DOT_UNROLL];
19401940

@@ -2208,19 +2208,19 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {
22082208
#endif
22092209
}
22102210

2211-
inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, x, x); *s = sqrt(*s); }
2211+
inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, x, x); *s = sqrtf(*s); }
22122212
inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }
2213-
inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrt(x[i]); }
2213+
inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }
22142214
inline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }
22152215
inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }
22162216
inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }
22172217
inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }
22182218

2219-
static const ggml_float GELU_COEF_A = 0.044715;
2220-
static const ggml_float SQRT_2_OVER_PI = 0.79788456080286535587989211986876;
2219+
static const float GELU_COEF_A = 0.044715f;
2220+
static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
22212221

22222222
inline static float ggml_gelu_f32(float x) {
2223-
return 0.5*x*(1.0 + tanh(SQRT_2_OVER_PI*x*(1.0 + GELU_COEF_A*x*x)));
2223+
return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
22242224
}
22252225

22262226
inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
@@ -2249,7 +2249,7 @@ inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
22492249

22502250
// Sigmoid Linear Unit (SiLU) function
22512251
inline static float ggml_silu_f32(float x) {
2252-
return x/(1.0 + exp(-x));
2252+
return x/(1.0f + expf(-x));
22532253
}
22542254

22552255
inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
@@ -2280,7 +2280,7 @@ inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
22802280
#ifndef GGML_USE_ACCELERATE
22812281
ggml_float sum = 0.0;
22822282
for (int i = 0; i < n; ++i) {
2283-
sum += x[i];
2283+
sum += (ggml_float)x[i];
22842284
}
22852285
*s = sum;
22862286
#else
@@ -2290,7 +2290,7 @@ inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
22902290

22912291
inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
22922292
#ifndef GGML_USE_ACCELERATE
2293-
ggml_float max = -INFINITY;
2293+
float max = -INFINITY;
22942294
for (int i = 0; i < n; ++i) {
22952295
max = MAX(max, x[i]);
22962296
}
@@ -2300,7 +2300,10 @@ inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
23002300
#endif
23012301
}
23022302

2303-
inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) { ggml_vec_norm_f32(n, s, x); *s = 1./(*s); }
2303+
inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {
2304+
ggml_vec_norm_f32(n, s, x);
2305+
*s = 1.f/(*s);
2306+
}
23042307

23052308
//
23062309
// logging
@@ -2695,7 +2698,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
26952698
const float f = table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(ii);
26962699
table_gelu_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_f32(f));
26972700
table_silu_f16[i] = GGML_FP32_TO_FP16(ggml_silu_f32(f));
2698-
table_exp_f16[i] = GGML_FP32_TO_FP16(exp(f));
2701+
table_exp_f16[i] = GGML_FP32_TO_FP16(expf(f));
26992702
}
27002703

27012704
const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
@@ -5737,31 +5740,32 @@ static void ggml_compute_forward_norm_f32(
57375740
const size_t nb2 = dst->nb[2];
57385741
const size_t nb3 = dst->nb[3];
57395742

5740-
const ggml_float eps = 1e-5f; // TODO: make this a parameter
5743+
const float eps = 1e-5f; // TODO: make this a parameter
57415744

57425745
// TODO: optimize
57435746
for (int i03 = 0; i03 < ne03; i03++) {
57445747
for (int i02 = 0; i02 < ne02; i02++) {
57455748
for (int i01 = ith; i01 < ne01; i01 += nth) {
57465749
const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
57475750

5748-
ggml_float mean = 0.0;
5751+
ggml_float sum = 0.0;
57495752
for (int i00 = 0; i00 < ne00; i00++) {
5750-
mean += x[i00];
5753+
sum += (ggml_float)x[i00];
57515754
}
57525755

5753-
mean /= ne00;
5756+
float mean = sum/ne00;
57545757

57555758
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
57565759

57575760
ggml_float sum2 = 0.0;
57585761
for (int i00 = 0; i00 < ne00; i00++) {
5759-
ggml_float v = x[i00] - mean;
5762+
float v = x[i00] - mean;
57605763
y[i00] = v;
5761-
sum2 += v*v;
5764+
sum2 += (ggml_float)(v*v);
57625765
}
57635766

5764-
const float scale = 1.0/sqrt(sum2/ne00 + eps);
5767+
float variance = sum2/ne00;
5768+
const float scale = 1.0f/sqrtf(variance + eps);
57655769

57665770
ggml_vec_scale_f32(ne00, y, scale);
57675771
}
@@ -5819,20 +5823,20 @@ static void ggml_compute_forward_rms_norm_f32(
58195823
const size_t nb2 = dst->nb[2];
58205824
const size_t nb3 = dst->nb[3];
58215825

5822-
const ggml_float eps = 1e-6f; // TODO: make this a parameter
5826+
const float eps = 1e-6f; // TODO: make this a parameter
58235827

58245828
// TODO: optimize
58255829
for (int i03 = 0; i03 < ne03; i03++) {
58265830
for (int i02 = 0; i02 < ne02; i02++) {
58275831
for (int i01 = ith; i01 < ne01; i01 += nth) {
58285832
const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
58295833

5830-
ggml_float mean = 0.0;
5834+
ggml_float sum = 0.0;
58315835
for (int i00 = 0; i00 < ne00; i00++) {
5832-
mean += x[i00] * x[i00];
5836+
sum += (ggml_float)(x[i00] * x[i00]);
58335837
}
58345838

5835-
mean /= ne00;
5839+
float mean = sum/ne00;
58365840

58375841
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
58385842

@@ -5841,7 +5845,7 @@ static void ggml_compute_forward_rms_norm_f32(
58415845
// y[i00] = x[i00];
58425846
// }
58435847

5844-
const float scale = 1.0/sqrt(mean + eps);
5848+
const float scale = 1.0f/sqrtf(mean + eps);
58455849

58465850
ggml_vec_scale_f32(ne00, y, scale);
58475851
}
@@ -7407,12 +7411,12 @@ static void ggml_compute_forward_soft_max_f32(
74077411
ggml_fp16_t s = GGML_FP32_TO_FP16(p[i] - max);
74087412
memcpy(&scvt, &s, sizeof(scvt));
74097413
const float val = GGML_FP16_TO_FP32(table_exp_f16[scvt]);
7410-
sum += val;
7414+
sum += (ggml_float)val;
74117415
p[i] = val;
74127416
}
74137417
}
74147418

7415-
assert(sum > 0.0f);
7419+
assert(sum > 0.0);
74167420

74177421
sum = 1.0/sum;
74187422
ggml_vec_scale_f32(nc, p, sum);
@@ -7496,8 +7500,8 @@ static void ggml_compute_forward_rope_f32(
74967500
const float * const src = (float *)((char *) src0->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
74977501
float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
74987502

7499-
double x0 = src[0];
7500-
double x1 = src[1];
7503+
double x0 = (double)src[0];
7504+
double x1 = (double)src[1];
75017505

75027506
dst_data[0] = x0*cos_theta - x1*sin_theta;
75037507
dst_data[1] = x0*sin_theta + x1*cos_theta;
@@ -7552,8 +7556,8 @@ static void ggml_compute_forward_rope_f16(
75527556
const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
75537557
ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
75547558

7555-
double x0 = ggml_fp16_to_fp32(src[0]);
7556-
double x1 = ggml_fp16_to_fp32(src[1]);
7559+
double x0 = (double)ggml_fp16_to_fp32(src[0]);
7560+
double x1 = (double)ggml_fp16_to_fp32(src[1]);
75577561

75587562
dst_data[0] = ggml_fp32_to_fp16(x0*cos_theta - x1*sin_theta);
75597563
dst_data[1] = ggml_fp32_to_fp16(x0*sin_theta + x1*cos_theta);
@@ -8229,7 +8233,7 @@ static void ggml_compute_forward_flash_attn_f32(
82298233
const int ir0 = dr*ith;
82308234
const int ir1 = MIN(ir0 + dr, nr);
82318235

8232-
const float scale = 1.0/sqrt((double) D);
8236+
const float scale = 1.0f/sqrtf(D);
82338237

82348238
//printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
82358239

@@ -8276,7 +8280,7 @@ static void ggml_compute_forward_flash_attn_f32(
82768280
float max = -INFINITY;
82778281
ggml_vec_max_f32(M, &max, S);
82788282

8279-
float sum = 0.0f;
8283+
ggml_float sum = 0.0;
82808284
{
82818285
#ifdef GGML_SOFT_MAX_ACCELERATE
82828286
max = -max;
@@ -8297,7 +8301,7 @@ static void ggml_compute_forward_flash_attn_f32(
82978301
ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);
82988302
memcpy(&scvt[j], &s, sizeof(uint16_t));
82998303
const float val = GGML_FP16_TO_FP32(table_exp_f16[scvt[j]]);
8300-
sump[j] += val;
8304+
sump[j] += (ggml_float)val;
83018305
SS[j] = val;
83028306
}
83038307
}
@@ -8309,7 +8313,7 @@ static void ggml_compute_forward_flash_attn_f32(
83098313
#endif
83108314
}
83118315

8312-
assert(sum > 0.0f);
8316+
assert(sum > 0.0);
83138317

83148318
sum = 1.0/sum;
83158319
ggml_vec_scale_f32(M, S, sum);
@@ -8438,7 +8442,7 @@ static void ggml_compute_forward_flash_attn_f16(
84388442
const int ir0 = dr*ith;
84398443
const int ir1 = MIN(ir0 + dr, nr);
84408444

8441-
const float scale = 1.0/sqrt((double) D);
8445+
const float scale = 1.0f/sqrtf(D);
84428446

84438447
//printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
84448448

@@ -8502,7 +8506,7 @@ static void ggml_compute_forward_flash_attn_f16(
85028506
float max = -INFINITY;
85038507
ggml_vec_max_f32(M, &max, S);
85048508

8505-
float sum = 0.0f;
8509+
ggml_float sum = 0.0;
85068510
{
85078511
#ifdef GGML_SOFT_MAX_ACCELERATE
85088512
max = -max;
@@ -8523,7 +8527,7 @@ static void ggml_compute_forward_flash_attn_f16(
85238527
ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);
85248528
memcpy(&scvt[j], &s, sizeof(uint16_t));
85258529
const float val = GGML_FP16_TO_FP32(table_exp_f16[scvt[j]]);
8526-
sump[j] += val;
8530+
sump[j] += (ggml_float)val;
85278531
SS[j] = val;
85288532
}
85298533
}
@@ -8535,7 +8539,7 @@ static void ggml_compute_forward_flash_attn_f16(
85358539
#endif
85368540
}
85378541

8538-
assert(sum > 0.0f);
8542+
assert(sum > 0.0);
85398543

85408544
sum = 1.0/sum;
85418545
ggml_vec_scale_f32(M, S, sum);
@@ -10066,7 +10070,7 @@ label=\"%d [%d, %d] | <x>%s",
1006610070
fprintf(fp, " \"%p\" [ \
1006710071
style = filled; fillcolor = %s; shape = record; \
1006810072
label=\"<x>%.1e\"; ]\n",
10069-
(void *) node, color, ggml_get_f32_1d(node, 0));
10073+
(void *) node, color, (double)ggml_get_f32_1d(node, 0));
1007010074
} else {
1007110075
fprintf(fp, " \"%p\" [ \
1007210076
style = filled; fillcolor = %s; shape = record; \
@@ -10304,7 +10308,7 @@ static enum ggml_opt_result ggml_opt_adam(
1030410308
if (params.past <= t) {
1030510309
const float rate = (pf[t%params.past] - fx)/fx;
1030610310

10307-
if (fabs(rate) < params.delta) {
10311+
if (fabsf(rate) < params.delta) {
1030810312
return GGML_OPT_OK;
1030910313
}
1031010314
}
@@ -10383,7 +10387,7 @@ static enum ggml_opt_result linesearch_backtracking(
1038310387
const float dec = 0.5f;
1038410388
const float inc = 2.1f;
1038510389

10386-
if (*step <= 0.) {
10390+
if (*step <= 0.f) {
1038710391
return GGML_LINESEARCH_INVALID_PARAMETERS;
1038810392
}
1038910393

@@ -10471,7 +10475,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(
1047110475
struct ggml_cgraph * gb) {
1047210476
if (params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE ||
1047310477
params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE) {
10474-
if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1. <= params.lbfgs.wolfe) {
10478+
if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1.f <= params.lbfgs.wolfe) {
1047510479
return GGML_OPT_INVALID_WOLFE;
1047610480
}
1047710481
}
@@ -10592,8 +10596,8 @@ static enum ggml_opt_result ggml_opt_lbfgs(
1059210596

1059310597
GGML_PRINT_DEBUG("f = %10.6f\n", ggml_get_f32_1d(f, 0));
1059410598

10595-
if (xnorm < 1.0) {
10596-
xnorm = 1.0;
10599+
if (xnorm < 1.0f) {
10600+
xnorm = 1.0f;
1059710601
}
1059810602
if (gnorm/xnorm <= params.lbfgs.eps) {
1059910603
// converged
@@ -10606,7 +10610,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(
1060610610
if (params.past <= k) {
1060710611
const float rate = (pf[k%params.past] - fx)/fx;
1060810612

10609-
if (fabs(rate) < params.delta) {
10613+
if (fabsf(rate) < params.delta) {
1061010614
return GGML_OPT_OK;
1061110615
}
1061210616
}

0 commit comments

Comments
 (0)