58 Star 280 Fork 3

腾讯开源/ncnn

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
testutil.cpp 47.95 KB
一键复制 编辑 原始数据 按行查看 历史
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
#include "testutil.h"
#include "cpu.h"
#include "layer.h"
#include "mat.h"
#include "prng.h"
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#if NCNN_VULKAN
#include "command.h"
#include "gpu.h"
#endif // NCNN_VULKAN
static struct prng_rand_t g_prng_rand_state;
void SRAND(int seed)
{
prng_srand(seed, &g_prng_rand_state);
}
uint64_t RAND()
{
return prng_rand(&g_prng_rand_state);
}
float RandomFloat(float a, float b)
{
float random = ((float)RAND()) / (float)uint64_t(-1); //RAND_MAX;
float diff = b - a;
float r = random * diff;
float v = a + r;
// generate denormal as zero
if (v < 0.0001 && v > -0.0001)
v = 0.f;
return v;
}
int RandomInt(int a, int b)
{
float random = ((float)RAND()) / (float)uint64_t(-1); //RAND_MAX;
int diff = b - a;
float r = random * diff;
return a + (int)r;
}
signed char RandomS8()
{
return (signed char)RandomInt(-127, 127);
}
void Randomize(ncnn::Mat& m, float a, float b)
{
for (size_t i = 0; i < m.total(); i++)
{
m[i] = RandomFloat(a, b);
}
}
void RandomizeInt(ncnn::Mat& m, int a, int b)
{
for (size_t i = 0; i < m.total(); i++)
{
((int*)m)[i] = RandomInt(a, b);
}
}
void RandomizeS8(ncnn::Mat& m)
{
for (size_t i = 0; i < m.total(); i++)
{
((signed char*)m)[i] = RandomS8();
}
}
ncnn::Mat RandomMat(int w, float a, float b)
{
ncnn::Mat m(w);
Randomize(m, a, b);
return m;
}
ncnn::Mat RandomMat(int w, int h, float a, float b)
{
ncnn::Mat m(w, h);
Randomize(m, a, b);
return m;
}
ncnn::Mat RandomMat(int w, int h, int c, float a, float b)
{
ncnn::Mat m(w, h, c);
Randomize(m, a, b);
return m;
}
ncnn::Mat RandomMat(int w, int h, int d, int c, float a, float b)
{
ncnn::Mat m(w, h, d, c);
Randomize(m, a, b);
return m;
}
ncnn::Mat RandomIntMat(int w)
{
ncnn::Mat m(w);
RandomizeInt(m);
return m;
}
ncnn::Mat RandomIntMat(int w, int h)
{
ncnn::Mat m(w, h);
RandomizeInt(m);
return m;
}
ncnn::Mat RandomIntMat(int w, int h, int c)
{
ncnn::Mat m(w, h, c);
RandomizeInt(m);
return m;
}
ncnn::Mat RandomIntMat(int w, int h, int d, int c)
{
ncnn::Mat m(w, h, d, c);
RandomizeInt(m);
return m;
}
ncnn::Mat RandomS8Mat(int w)
{
ncnn::Mat m(w, (size_t)1u);
RandomizeS8(m);
return m;
}
ncnn::Mat RandomS8Mat(int w, int h)
{
ncnn::Mat m(w, h, (size_t)1u);
RandomizeS8(m);
return m;
}
ncnn::Mat RandomS8Mat(int w, int h, int c)
{
ncnn::Mat m(w, h, c, (size_t)1u);
RandomizeS8(m);
return m;
}
ncnn::Mat RandomS8Mat(int w, int h, int d, int c)
{
ncnn::Mat m(w, h, d, c, (size_t)1u);
RandomizeS8(m);
return m;
}
ncnn::Mat scales_mat(const ncnn::Mat& mat, int m, int k, int ldx)
{
ncnn::Mat weight_scales(m);
for (int i = 0; i < m; ++i)
{
float min = mat[0], _max = mat[0];
const float* ptr = (const float*)(mat.data) + i * ldx;
for (int j = 0; j < k; ++j)
{
if (min > ptr[j])
{
min = ptr[j];
}
if (_max < ptr[j])
{
_max = ptr[j];
}
}
const float abs_min = abs(min), abs_max = abs(_max);
weight_scales[i] = 127.f / (abs_min > abs_max ? abs_min : abs_max);
}
return weight_scales;
}
bool NearlyEqual(float a, float b, float epsilon)
{
if (a == b)
return true;
float diff = (float)fabs(a - b);
if (diff <= epsilon)
return true;
// relative error
return diff < epsilon * std::max(fabs(a), fabs(b));
}
int Compare(const ncnn::Mat& a, const ncnn::Mat& b, float epsilon)
{
#define CHECK_MEMBER(m) \
if (a.m != b.m) \
{ \
fprintf(stderr, #m " not match expect %d but got %d\n", (int)a.m, (int)b.m); \
return -1; \
}
CHECK_MEMBER(dims)
CHECK_MEMBER(w)
CHECK_MEMBER(h)
CHECK_MEMBER(d)
CHECK_MEMBER(c)
CHECK_MEMBER(elemsize)
CHECK_MEMBER(elempack)
#undef CHECK_MEMBER
for (int q = 0; q < a.c; q++)
{
const ncnn::Mat ma = a.channel(q);
const ncnn::Mat mb = b.channel(q);
for (int z = 0; z < a.d; z++)
{
const ncnn::Mat da = ma.depth(z);
const ncnn::Mat db = mb.depth(z);
for (int i = 0; i < a.h; i++)
{
const float* pa = da.row(i);
const float* pb = db.row(i);
for (int j = 0; j < a.w; j++)
{
if (!NearlyEqual(pa[j], pb[j], epsilon))
{
fprintf(stderr, "value not match at c:%d d:%d h:%d w:%d expect %f but got %f\n", q, z, i, j, pa[j], pb[j]);
return -1;
}
}
}
}
}
return 0;
}
int CompareMat(const ncnn::Mat& a, const ncnn::Mat& b, float epsilon)
{
ncnn::Option opt;
opt.num_threads = 1;
if (a.elempack != 1)
{
ncnn::Mat a1;
ncnn::convert_packing(a, a1, 1, opt);
return CompareMat(a1, b, epsilon);
}
if (b.elempack != 1)
{
ncnn::Mat b1;
ncnn::convert_packing(b, b1, 1, opt);
return CompareMat(a, b1, epsilon);
}
if (a.elemsize == 2u)
{
ncnn::Mat a32;
cast_float16_to_float32(a, a32, opt);
return CompareMat(a32, b, epsilon);
}
if (a.elemsize == 1u)
{
ncnn::Mat a32;
cast_int8_to_float32(a, a32, opt);
return CompareMat(a32, b, epsilon);
}
if (b.elemsize == 2u)
{
ncnn::Mat b32;
cast_float16_to_float32(b, b32, opt);
return CompareMat(a, b32, epsilon);
}
if (b.elemsize == 1u)
{
ncnn::Mat b32;
cast_int8_to_float32(b, b32, opt);
return CompareMat(a, b32, epsilon);
}
return Compare(a, b, epsilon);
}
int CompareMat(const std::vector<ncnn::Mat>& a, const std::vector<ncnn::Mat>& b, float epsilon)
{
if (a.size() != b.size())
{
fprintf(stderr, "output blob count not match %zu %zu\n", a.size(), b.size());
return -1;
}
for (size_t i = 0; i < a.size(); i++)
{
if (CompareMat(a[i], b[i], epsilon))
{
fprintf(stderr, "output blob %zu not match\n", i);
return -1;
}
}
return 0;
}
static int convert_to_optimal_layout(const ncnn::Mat& a, ncnn::Mat& a4, const ncnn::Option& opt, const ncnn::Layer* op, int flag)
{
// clang-format off
// *INDENT-OFF*
#if NCNN_ARM82
if (opt.use_fp16_storage && ncnn::cpu_support_arm_asimdhp() && op->support_fp16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::cast_float32_to_float16(a, a4, opt);
}
else
#endif // NCNN_ARM82
#if NCNN_VFPV4
if (opt.use_fp16_storage && !opt.use_bf16_storage && ncnn::cpu_support_arm_vfpv4() && op->support_fp16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::cast_float32_to_float16(a, a4, opt);
}
else
#endif // NCNN_VFPV4
#if NCNN_ZFH
if (opt.use_fp16_storage && (ncnn::cpu_support_riscv_zvfh() || (!ncnn::cpu_support_riscv_v() && ncnn::cpu_support_riscv_zfh())) && op->support_fp16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::cast_float32_to_float16(a, a4, opt);
}
else
#endif // NCNN_ZFH
#if NCNN_BF16
if (opt.use_bf16_storage && op->support_bf16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::cast_float32_to_bfloat16(a, a4, opt);
}
else
#endif // NCNN_BF16
if (opt.use_fp16_storage && op->support_fp16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::cast_float32_to_float16(a, a4, opt);
}
else
{
a4 = a;
}
// *INDENT-ON*
// clang-format on
if (opt.use_packing_layout && op->support_packing && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_PACKING))
{
// resolve dst_elempack
int dims = a4.dims;
int elemcount = 0;
if (dims == 1) elemcount = a4.elempack * a4.w;
if (dims == 2) elemcount = a4.elempack * a4.h;
if (dims == 3 || dims == 4) elemcount = a4.elempack * a4.c;
int elembits = a4.elembits();
int dst_elempack = 1;
if (elembits == 32)
{
#if NCNN_AVX512
if (elemcount % 16 == 0 && ncnn::cpu_support_x86_avx512())
dst_elempack = 16;
else if (elemcount % 8 == 0 && ncnn::cpu_support_x86_avx())
dst_elempack = 8;
else if (elemcount % 4 == 0)
dst_elempack = 4;
#elif NCNN_AVX
if (elemcount % 8 == 0 && ncnn::cpu_support_x86_avx())
dst_elempack = 8;
else if (elemcount % 4 == 0)
dst_elempack = 4;
#elif NCNN_RVV || NCNN_XTHEADVECTOR
const int packn = ncnn::cpu_riscv_vlenb() / 4;
if (elemcount % packn == 0)
dst_elempack = packn;
#else
if (elemcount % 4 == 0)
dst_elempack = 4;
#endif
}
if (elembits == 16)
{
#if NCNN_ARM82
if (elemcount % 8 == 0 && ncnn::cpu_support_arm_asimdhp() && opt.use_fp16_arithmetic && op->support_fp16_storage)
dst_elempack = 8;
else if (elemcount % 4 == 0)
dst_elempack = 4;
#elif NCNN_RVV || NCNN_XTHEADVECTOR
const int packn = ncnn::cpu_riscv_vlenb() / 2;
if (elemcount % packn == 0)
dst_elempack = packn;
#else
if (elemcount % 4 == 0)
dst_elempack = 4;
#endif
}
if (elembits == 8)
{
#if NCNN_RVV || NCNN_XTHEADVECTOR
const int packn = ncnn::cpu_riscv_vlenb() / 1;
if (elemcount % packn == 0)
dst_elempack = packn;
#else
if (elemcount % 8 == 0)
dst_elempack = 8;
#endif
}
if (flag & TEST_LAYER_ENABLE_FORCE_INPUT_PACK8)
dst_elempack = 8;
ncnn::Mat a4_packed;
ncnn::convert_packing(a4, a4_packed, dst_elempack, opt);
a4 = a4_packed;
}
return 0;
}
static int convert_to_vanilla_layout(const ncnn::Mat& c4, ncnn::Mat& c, const ncnn::Option& opt, const ncnn::Layer* op, int flag)
{
ncnn::Mat c4_unpacked;
if (c4.elempack != 1)
{
ncnn::convert_packing(c4, c4_unpacked, 1, opt);
}
else
{
c4_unpacked = c4;
}
// clang-format off
// *INDENT-OFF*
#if NCNN_ARM82
if (opt.use_fp16_storage && ncnn::cpu_support_arm_asimdhp() && op->support_fp16_storage && c4_unpacked.elembits() == 16)
{
ncnn::cast_float16_to_float32(c4_unpacked, c, opt);
}
else
#endif // NCNN_ARM82
#if NCNN_VFPV4
if (opt.use_fp16_storage && !opt.use_bf16_storage && ncnn::cpu_support_arm_vfpv4() && op->support_fp16_storage && c4_unpacked.elembits() == 16)
{
ncnn::cast_float16_to_float32(c4_unpacked, c, opt);
}
else
#endif // NCNN_VFPV4
#if NCNN_ZFH
if (opt.use_fp16_storage && (ncnn::cpu_support_riscv_zvfh() || (!ncnn::cpu_support_riscv_v() && ncnn::cpu_support_riscv_zfh())) && op->support_fp16_storage && c4_unpacked.elembits() == 16)
{
ncnn::cast_float16_to_float32(c4_unpacked, c, opt);
}
else
#endif // NCNN_ZFH
#if NCNN_BF16
if (opt.use_bf16_storage && op->support_bf16_storage && c4_unpacked.elembits() == 16)
{
ncnn::cast_bfloat16_to_float32(c4_unpacked, c, opt);
}
else
#endif // NCNN_BF16
if (opt.use_fp16_storage && op->support_fp16_storage && c4_unpacked.elembits() == 16)
{
ncnn::cast_float16_to_float32(c4_unpacked, c, opt);
}
else
{
c = c4_unpacked;
}
// *INDENT-ON*
// clang-format on
return 0;
}
int test_layer_naive(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const std::vector<ncnn::Mat>& a, int top_blob_count, std::vector<ncnn::Mat>& b, void (*func)(ncnn::Layer*), int flag)
{
ncnn::Layer* op = ncnn::create_layer_naive(typeindex);
if (func)
{
(*func)((ncnn::Layer*)op);
}
op->load_param(pd);
if (op->one_blob_only && a.size() != 1)
{
fprintf(stderr, "layer with one_blob_only but consume multiple inputs\n");
delete op;
return -1;
}
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::Option opt;
opt.num_threads = 1;
opt.lightmode = false;
opt.use_packing_layout = false;
opt.use_fp16_packed = false;
opt.use_fp16_storage = false;
opt.use_fp16_arithmetic = false;
opt.use_shader_pack8 = false;
opt.use_bf16_storage = false;
opt.use_vulkan_compute = false;
op->create_pipeline(opt);
b.resize(top_blob_count);
if (op->support_inplace)
{
for (size_t i = 0; i < a.size(); i++)
{
b[i] = a[i].clone();
}
op->forward_inplace(b, opt);
}
else
{
op->forward(a, b, opt);
}
op->destroy_pipeline(opt);
delete op;
return 0;
}
int test_layer_cpu(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const std::vector<ncnn::Mat>& a, int top_blob_count, std::vector<ncnn::Mat>& c, const std::vector<ncnn::Mat>& top_shapes, void (*func)(ncnn::Layer*), int flag)
{
ncnn::Layer* op = ncnn::create_layer_cpu(typeindex);
if (!op->support_packing && _opt.use_packing_layout)
{
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
delete op;
return 233;
}
if (func)
{
(*func)((ncnn::Layer*)op);
}
if (!top_shapes.empty())
{
op->bottom_shapes = a;
op->top_shapes = top_shapes;
}
op->load_param(pd);
if (op->one_blob_only && a.size() != 1)
{
fprintf(stderr, "layer with one_blob_only but consume multiple inputs\n");
delete op;
return -1;
}
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::Option opt = _opt;
opt.num_threads = 1;
opt.use_vulkan_compute = false;
op->create_pipeline(opt);
if (!op->support_packing && _opt.use_packing_layout)
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
std::vector<ncnn::Mat> a4(a.size());
for (size_t i = 0; i < a4.size(); i++)
{
convert_to_optimal_layout(a[i], a4[i], opt, op, flag);
}
c.resize(top_blob_count);
if (op->support_inplace)
{
for (size_t i = 0; i < a4.size(); i++)
{
c[i] = a4[i].clone();
}
op->forward_inplace(c, opt);
}
else
{
op->forward(a4, c, opt);
}
for (size_t i = 0; i < c.size(); i++)
{
convert_to_vanilla_layout(c[i], c[i], opt, op, flag);
}
op->destroy_pipeline(opt);
delete op;
return 0;
}
#if NCNN_VULKAN
int test_layer_gpu(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const std::vector<ncnn::Mat>& a, int top_blob_count, std::vector<ncnn::Mat>& d, const std::vector<ncnn::Mat>& top_shapes, void (*func)(ncnn::Layer*), int flag)
{
if (!_opt.use_packing_layout)
{
// pack1 test is useless for gpu
return 233;
}
ncnn::Layer* op = ncnn::create_layer_vulkan(typeindex);
if (!op)
{
return 233;
}
op->load_param(pd);
if (!op->support_vulkan)
{
delete op;
return 233;
}
ncnn::VulkanDevice* vkdev = ncnn::get_gpu_device();
op->vkdev = vkdev;
if (func)
{
(*func)((ncnn::Layer*)op);
}
if (!top_shapes.empty())
{
op->bottom_shapes = a;
op->top_shapes = top_shapes;
}
if (op->one_blob_only && a.size() != 1)
{
fprintf(stderr, "layer with one_blob_only but consume multiple inputs\n");
delete op;
return -1;
}
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::VkWeightAllocator g_weight_vkallocator(vkdev);
ncnn::VkWeightStagingAllocator g_weight_staging_vkallocator(vkdev);
ncnn::VkAllocator* blob_vkallocator = vkdev->acquire_blob_allocator();
ncnn::VkAllocator* staging_vkallocator = vkdev->acquire_staging_allocator();
ncnn::Option opt = _opt;
opt.num_threads = 1;
opt.use_vulkan_compute = true;
opt.blob_vkallocator = blob_vkallocator;
opt.workspace_vkallocator = blob_vkallocator;
opt.staging_vkallocator = staging_vkallocator;
if (!vkdev->info.support_fp16_packed()) opt.use_fp16_packed = false;
if (!vkdev->info.support_fp16_storage()) opt.use_fp16_storage = false;
if (!vkdev->info.support_fp16_uniform()) opt.use_fp16_uniform = false;
if (!vkdev->info.support_fp16_arithmetic()) opt.use_fp16_arithmetic = false;
if (!vkdev->info.support_int8_packed()) opt.use_int8_packed = false;
if (!vkdev->info.support_int8_storage()) opt.use_int8_storage = false;
if (!vkdev->info.support_int8_uniform()) opt.use_int8_uniform = false;
if (!vkdev->info.support_int8_arithmetic()) opt.use_int8_arithmetic = false;
if (!vkdev->info.support_cooperative_matrix()) opt.use_cooperative_matrix = false;
if (!vkdev->info.support_subgroup_ops()) opt.use_subgroup_ops = false;
// FIXME fp16a may produce large error
opt.use_fp16_arithmetic = false;
op->create_pipeline(opt);
if (!op->support_vulkan)
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
{
ncnn::VkTransfer cmd(vkdev);
ncnn::Option opt_upload = opt;
opt_upload.blob_vkallocator = &g_weight_vkallocator;
opt_upload.workspace_vkallocator = &g_weight_vkallocator;
opt_upload.staging_vkallocator = &g_weight_staging_vkallocator;
op->upload_model(cmd, opt_upload);
cmd.submit_and_wait();
}
d.resize(top_blob_count);
{
// forward
ncnn::VkCompute cmd(vkdev);
{
// upload
std::vector<ncnn::VkMat> a_gpu(a.size());
for (size_t i = 0; i < a_gpu.size(); i++)
{
if (flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING)
{
// resolve dst_elempack
int dims = a[i].dims;
int elemcount = 0;
if (dims == 1) elemcount = a[i].elempack * a[i].w;
if (dims == 2) elemcount = a[i].elempack * a[i].h;
if (dims == 3 || dims == 4) elemcount = a[i].elempack * a[i].c;
const int dst_elempack = (opt.use_shader_pack8 && elemcount % 8 == 0) ? 8 : elemcount % 4 == 0 ? 4 : 1;
ncnn::Mat a4;
ncnn::convert_packing(a[i], a4, dst_elempack, opt);
ncnn::Option opt_upload = opt;
opt_upload.use_fp16_packed = false;
opt_upload.use_fp16_storage = false;
opt_upload.use_int8_packed = false;
opt_upload.use_int8_storage = false;
cmd.record_clone(a4, a_gpu[i], opt_upload);
}
else
{
cmd.record_upload(a[i], a_gpu[i], opt);
}
}
std::vector<ncnn::VkMat> d_gpu(top_blob_count);
if (op->support_inplace)
{
op->forward_inplace(a_gpu, cmd, opt);
d_gpu = a_gpu;
}
else
{
op->forward(a_gpu, d_gpu, cmd, opt);
}
// download
for (size_t i = 0; i < d_gpu.size(); i++)
{
cmd.record_download(d_gpu[i], d[i], opt);
}
}
cmd.submit_and_wait();
}
op->destroy_pipeline(opt);
delete op;
vkdev->reclaim_blob_allocator(blob_vkallocator);
vkdev->reclaim_staging_allocator(staging_vkallocator);
g_weight_vkallocator.clear();
g_weight_staging_vkallocator.clear();
return 0;
}
#endif // NCNN_VULKAN
int test_layer(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const std::vector<ncnn::Mat>& a, int top_blob_count, const std::vector<ncnn::Mat>& top_shapes, float epsilon, void (*func)(ncnn::Layer*), int flag)
{
// naive
std::vector<ncnn::Mat> b;
{
int ret = test_layer_naive(typeindex, pd, weights, a, top_blob_count, b, func, flag);
if (ret != 233 && ret != 0)
{
fprintf(stderr, "test_layer_naive failed\n");
return -1;
}
}
// cpu
{
std::vector<ncnn::Mat> c;
int ret = test_layer_cpu(typeindex, pd, weights, _opt, a, top_blob_count, c, std::vector<ncnn::Mat>(), func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, c, epsilon) != 0))
{
fprintf(stderr, "test_layer_cpu failed\n");
return -1;
}
}
// cpu shape hint
{
std::vector<ncnn::Mat> c;
int ret = test_layer_cpu(typeindex, pd, weights, _opt, a, top_blob_count, c, b, func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, c, epsilon) != 0))
{
fprintf(stderr, "test_layer_cpu failed with shape hint\n");
return -1;
}
}
#if NCNN_VULKAN
// gpu
if (!(flag & TEST_LAYER_DISABLE_GPU_TESTING))
{
std::vector<ncnn::Mat> d;
int ret = test_layer_gpu(typeindex, pd, weights, _opt, a, top_blob_count, d, std::vector<ncnn::Mat>(), func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, d, epsilon) != 0))
{
fprintf(stderr, "test_layer_gpu failed\n");
return -1;
}
}
// gpu shape hint
if (!(flag & TEST_LAYER_DISABLE_GPU_TESTING))
{
std::vector<ncnn::Mat> d;
int ret = test_layer_gpu(typeindex, pd, weights, _opt, a, top_blob_count, d, b, func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, d, epsilon) != 0))
{
fprintf(stderr, "test_layer_gpu failed with shape hint\n");
return -1;
}
}
#endif // NCNN_VULKAN
return 0;
}
int test_layer_naive(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Mat& a, ncnn::Mat& b, void (*func)(ncnn::Layer*), int flag)
{
ncnn::Layer* op = ncnn::create_layer_naive(typeindex);
if (func)
{
(*func)((ncnn::Layer*)op);
}
op->load_param(pd);
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::Option opt;
opt.num_threads = 1;
opt.lightmode = false;
opt.use_packing_layout = false;
opt.use_fp16_packed = false;
opt.use_fp16_storage = false;
opt.use_fp16_arithmetic = false;
opt.use_shader_pack8 = false;
opt.use_bf16_storage = false;
opt.use_vulkan_compute = false;
op->create_pipeline(opt);
if (op->support_inplace)
{
b = a.clone();
op->forward_inplace(b, opt);
}
else
{
op->forward(a, b, opt);
}
op->destroy_pipeline(opt);
delete op;
return 0;
}
int test_layer_cpu(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const ncnn::Mat& a, ncnn::Mat& c, const ncnn::Mat& top_shape, void (*func)(ncnn::Layer*), int flag)
{
ncnn::Layer* op = ncnn::create_layer_cpu(typeindex);
if (!op->support_packing && _opt.use_packing_layout)
{
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
delete op;
return 233;
}
if (func)
{
(*func)((ncnn::Layer*)op);
}
if (top_shape.dims)
{
op->bottom_shapes.resize(1);
op->top_shapes.resize(1);
op->bottom_shapes[0] = a;
op->top_shapes[0] = top_shape;
}
op->load_param(pd);
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::Option opt = _opt;
opt.num_threads = 1;
opt.use_vulkan_compute = false;
op->create_pipeline(opt);
if (!op->support_packing && _opt.use_packing_layout)
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
ncnn::Mat a4;
convert_to_optimal_layout(a, a4, opt, op, flag);
if (op->support_inplace)
{
c = a4.clone();
op->forward_inplace(c, opt);
}
else
{
op->forward(a4, c, opt);
}
convert_to_vanilla_layout(c, c, opt, op, flag);
op->destroy_pipeline(opt);
delete op;
return 0;
}
#if NCNN_VULKAN
int test_layer_gpu(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const ncnn::Mat& a, ncnn::Mat& d, const ncnn::Mat& top_shape, void (*func)(ncnn::Layer*), int flag)
{
if (!_opt.use_packing_layout)
{
// pack1 test is useless for gpu
return 233;
}
ncnn::Layer* op = ncnn::create_layer_vulkan(typeindex);
if (!op)
{
return 233;
}
op->load_param(pd);
if (!op->support_vulkan)
{
delete op;
return 233;
}
ncnn::VulkanDevice* vkdev = ncnn::get_gpu_device();
op->vkdev = vkdev;
if (func)
{
(*func)((ncnn::Layer*)op);
}
if (top_shape.dims)
{
op->bottom_shapes.resize(1);
op->top_shapes.resize(1);
op->bottom_shapes[0] = a;
op->top_shapes[0] = top_shape;
}
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::VkWeightAllocator g_weight_vkallocator(vkdev);
ncnn::VkWeightStagingAllocator g_weight_staging_vkallocator(vkdev);
ncnn::VkAllocator* blob_vkallocator = vkdev->acquire_blob_allocator();
ncnn::VkAllocator* staging_vkallocator = vkdev->acquire_staging_allocator();
ncnn::Option opt = _opt;
opt.num_threads = 1;
opt.use_vulkan_compute = true;
opt.blob_vkallocator = blob_vkallocator;
opt.workspace_vkallocator = blob_vkallocator;
opt.staging_vkallocator = staging_vkallocator;
if (!vkdev->info.support_fp16_packed()) opt.use_fp16_packed = false;
if (!vkdev->info.support_fp16_storage()) opt.use_fp16_storage = false;
if (!vkdev->info.support_fp16_uniform()) opt.use_fp16_uniform = false;
if (!vkdev->info.support_fp16_arithmetic()) opt.use_fp16_arithmetic = false;
if (!vkdev->info.support_int8_packed()) opt.use_int8_packed = false;
if (!vkdev->info.support_int8_storage()) opt.use_int8_storage = false;
if (!vkdev->info.support_int8_uniform()) opt.use_int8_uniform = false;
if (!vkdev->info.support_int8_arithmetic()) opt.use_int8_arithmetic = false;
if (!vkdev->info.support_cooperative_matrix()) opt.use_cooperative_matrix = false;
if (!vkdev->info.support_subgroup_ops()) opt.use_subgroup_ops = false;
// FIXME fp16a may produce large error
opt.use_fp16_arithmetic = false;
op->create_pipeline(opt);
if (!op->support_vulkan)
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
{
ncnn::VkTransfer cmd(vkdev);
ncnn::Option opt_upload = opt;
opt_upload.blob_vkallocator = &g_weight_vkallocator;
opt_upload.workspace_vkallocator = &g_weight_vkallocator;
opt_upload.staging_vkallocator = &g_weight_staging_vkallocator;
op->upload_model(cmd, opt_upload);
cmd.submit_and_wait();
}
{
// forward
ncnn::VkCompute cmd(vkdev);
{
// upload
ncnn::VkMat a_gpu;
if (flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING)
{
// resolve dst_elempack
int dims = a.dims;
int elemcount = 0;
if (dims == 1) elemcount = a.elempack * a.w;
if (dims == 2) elemcount = a.elempack * a.h;
if (dims == 3 || dims == 4) elemcount = a.elempack * a.c;
const int dst_elempack = (opt.use_shader_pack8 && elemcount % 8 == 0) ? 8 : elemcount % 4 == 0 ? 4 : 1;
ncnn::Mat a4;
ncnn::convert_packing(a, a4, dst_elempack, opt);
ncnn::Option opt_upload = opt;
opt_upload.use_fp16_packed = false;
opt_upload.use_fp16_storage = false;
opt_upload.use_int8_packed = false;
opt_upload.use_int8_storage = false;
cmd.record_clone(a4, a_gpu, opt_upload);
}
else
{
cmd.record_upload(a, a_gpu, opt);
}
ncnn::VkMat d_gpu;
if (op->support_inplace)
{
op->forward_inplace(a_gpu, cmd, opt);
d_gpu = a_gpu;
}
else
{
op->forward(a_gpu, d_gpu, cmd, opt);
}
// download
cmd.record_download(d_gpu, d, opt);
}
cmd.submit_and_wait();
}
op->destroy_pipeline(opt);
delete op;
vkdev->reclaim_blob_allocator(blob_vkallocator);
vkdev->reclaim_staging_allocator(staging_vkallocator);
g_weight_vkallocator.clear();
g_weight_staging_vkallocator.clear();
return 0;
}
#endif // NCNN_VULKAN
int test_layer(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const ncnn::Mat& a, const ncnn::Mat& top_shape, float epsilon, void (*func)(ncnn::Layer*), int flag)
{
// naive
ncnn::Mat b;
{
int ret = test_layer_naive(typeindex, pd, weights, a, b, func, flag);
if (ret != 233 && ret != 0)
{
fprintf(stderr, "test_layer_naive failed\n");
return -1;
}
}
// cpu
{
ncnn::Mat c;
int ret = test_layer_cpu(typeindex, pd, weights, _opt, a, c, ncnn::Mat(), func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, c, epsilon) != 0))
{
fprintf(stderr, "test_layer_cpu failed\n");
return -1;
}
}
// cpu shape hint
{
ncnn::Mat c;
int ret = test_layer_cpu(typeindex, pd, weights, _opt, a, c, b, func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, c, epsilon) != 0))
{
fprintf(stderr, "test_layer_cpu failed with shape hint\n");
return -1;
}
}
#if NCNN_VULKAN
// gpu
if (!(flag & TEST_LAYER_DISABLE_GPU_TESTING))
{
ncnn::Mat d;
int ret = test_layer_gpu(typeindex, pd, weights, _opt, a, d, ncnn::Mat(), func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, d, epsilon) != 0))
{
fprintf(stderr, "test_layer_gpu failed\n");
return -1;
}
}
// gpu shape hint
if (!(flag & TEST_LAYER_DISABLE_GPU_TESTING))
{
ncnn::Mat d;
int ret = test_layer_gpu(typeindex, pd, weights, _opt, a, d, b, func, flag);
if (ret != 233 && (ret != 0 || CompareMat(b, d, epsilon) != 0))
{
fprintf(stderr, "test_layer_gpu failed with shape hint\n");
return -1;
}
}
#endif // NCNN_VULKAN
return 0;
}
int test_layer_opt(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& opt, const std::vector<ncnn::Mat>& a, int top_blob_count, float epsilon, void (*func)(ncnn::Layer*), int flag)
{
// fp16 representation
std::vector<ncnn::Mat> a_fp16;
if (opt.use_bf16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
a_fp16.resize(a.size());
for (size_t j = 0; j < a.size(); j++)
{
ncnn::Mat tmp;
ncnn::cast_float32_to_bfloat16(a[j], tmp, opt);
ncnn::cast_bfloat16_to_float32(tmp, a_fp16[j], opt);
}
}
else if ((opt.use_fp16_packed || opt.use_fp16_storage) && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
a_fp16.resize(a.size());
for (size_t j = 0; j < a.size(); j++)
{
ncnn::Mat tmp;
ncnn::cast_float32_to_float16(a[j], tmp, opt);
ncnn::cast_float16_to_float32(tmp, a_fp16[j], opt);
}
}
else
{
a_fp16 = a;
}
std::vector<ncnn::Mat> weights_fp16;
float epsilon_fp16;
if (opt.use_bf16_storage)
{
weights_fp16.resize(weights.size());
for (size_t j = 0; j < weights.size(); j++)
{
if (weights[j].elembits() != 32)
{
weights_fp16[j] = weights[j];
continue;
}
ncnn::Mat tmp;
ncnn::cast_float32_to_bfloat16(weights[j], tmp, opt);
ncnn::cast_bfloat16_to_float32(tmp, weights_fp16[j], opt);
}
epsilon_fp16 = epsilon * 100; // 0.1
}
else if (opt.use_fp16_packed || opt.use_fp16_storage)
{
weights_fp16.resize(weights.size());
for (size_t j = 0; j < weights.size(); j++)
{
if (weights[j].elembits() != 32)
{
weights_fp16[j] = weights[j];
continue;
}
ncnn::Mat tmp;
ncnn::cast_float32_to_float16(weights[j], tmp, opt);
ncnn::cast_float16_to_float32(tmp, weights_fp16[j], opt);
}
epsilon_fp16 = epsilon * 100; // 0.1
}
else
{
weights_fp16 = weights;
epsilon_fp16 = epsilon;
}
if (opt.use_fp16_arithmetic)
{
epsilon_fp16 = epsilon * 1000; // 1.0
}
std::vector<ncnn::Mat> top_shapes;
int ret = test_layer(ncnn::layer_to_index(layer_type), pd, weights_fp16, opt, a_fp16, top_blob_count, top_shapes, epsilon_fp16, func, flag);
if (ret != 0)
{
fprintf(stderr, "test_layer %s failed use_packing_layout=%d use_fp16_packed=%d use_fp16_storage=%d use_fp16_arithmetic=%d use_shader_pack8=%d use_bf16_storage=%d use_sgemm_convolution=%d use_winograd_convolution=%d\n", layer_type, opt.use_packing_layout, opt.use_fp16_packed, opt.use_fp16_storage, opt.use_fp16_arithmetic, opt.use_shader_pack8, opt.use_bf16_storage, opt.use_sgemm_convolution, opt.use_winograd_convolution);
return ret;
}
return 0;
}
int test_layer_opt(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& opt, const ncnn::Mat& a, float epsilon, void (*func)(ncnn::Layer*), int flag)
{
// fp16 representation
ncnn::Mat a_fp16;
if (opt.use_bf16_storage && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::Mat tmp;
ncnn::cast_float32_to_bfloat16(a, tmp, opt);
ncnn::cast_bfloat16_to_float32(tmp, a_fp16, opt);
}
else if ((opt.use_fp16_packed || opt.use_fp16_storage) && !(flag & TEST_LAYER_DISABLE_AUTO_INPUT_CASTING))
{
ncnn::Mat tmp;
ncnn::cast_float32_to_float16(a, tmp, opt);
ncnn::cast_float16_to_float32(tmp, a_fp16, opt);
}
else
{
a_fp16 = a;
}
std::vector<ncnn::Mat> weights_fp16;
float epsilon_fp16;
if (opt.use_bf16_storage)
{
weights_fp16.resize(weights.size());
for (size_t j = 0; j < weights.size(); j++)
{
if (weights[j].elembits() != 32)
{
weights_fp16[j] = weights[j];
continue;
}
ncnn::Mat tmp;
ncnn::cast_float32_to_bfloat16(weights[j], tmp, opt);
ncnn::cast_bfloat16_to_float32(tmp, weights_fp16[j], opt);
}
epsilon_fp16 = epsilon * 100; // 0.1
}
else if (opt.use_fp16_packed || opt.use_fp16_storage)
{
weights_fp16.resize(weights.size());
for (size_t j = 0; j < weights.size(); j++)
{
if (weights[j].elembits() != 32)
{
weights_fp16[j] = weights[j];
continue;
}
ncnn::Mat tmp;
ncnn::cast_float32_to_float16(weights[j], tmp, opt);
ncnn::cast_float16_to_float32(tmp, weights_fp16[j], opt);
}
epsilon_fp16 = epsilon * 100; // 0.1
}
else
{
weights_fp16 = weights;
epsilon_fp16 = epsilon;
}
if (opt.use_fp16_arithmetic)
{
epsilon_fp16 = epsilon * 1000; // 1.0
}
ncnn::Mat top_shape;
int ret = test_layer(ncnn::layer_to_index(layer_type), pd, weights_fp16, opt, a_fp16, top_shape, epsilon_fp16, func, flag);
if (ret != 0)
{
fprintf(stderr, "test_layer %s failed use_packing_layout=%d use_fp16_packed=%d use_fp16_storage=%d use_fp16_arithmetic=%d use_shader_pack8=%d use_bf16_storage=%d use_sgemm_convolution=%d use_winograd_convolution=%d\n", layer_type, opt.use_packing_layout, opt.use_fp16_packed, opt.use_fp16_storage, opt.use_fp16_arithmetic, opt.use_shader_pack8, opt.use_bf16_storage, opt.use_sgemm_convolution, opt.use_winograd_convolution);
return ret;
}
return 0;
}
int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const std::vector<ncnn::Mat>& a, int top_blob_count, float epsilon, void (*func)(ncnn::Layer*), int flag)
{
// pack fp16p fp16s fp16a bf16s shader8
const int options[][6] = {
{0, 0, 0, 0, 0, 0},
{0, 0, 1, 0, 0, 0},
{0, 0, 1, 1, 1, 0},
{1, 0, 0, 0, 0, 0},
{1, 1, 0, 0, 1, 0},
{1, 0, 1, 0, 0, 1},
{1, 1, 1, 1, 0, 0},
{1, 1, 1, 1, 1, 1},
};
const int opt_count = sizeof(options) / sizeof(options[0]);
for (int i = 0; i < opt_count; i++)
{
ncnn::Option opt;
opt.num_threads = 1;
opt.use_packing_layout = options[i][0];
opt.use_fp16_packed = options[i][1];
opt.use_fp16_storage = options[i][2];
opt.use_fp16_arithmetic = options[i][3];
opt.use_bf16_storage = options[i][4];
opt.use_shader_pack8 = options[i][5];
int ret = test_layer_opt(layer_type, pd, weights, opt, a, top_blob_count, epsilon, func, flag);
if (ret != 0)
return ret;
}
return 0;
}
int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Mat& a, float epsilon, void (*func)(ncnn::Layer*), int flag)
{
// pack fp16p fp16s fp16a bf16s shader8
const int options[][6] = {
{0, 0, 0, 0, 0, 0},
{0, 0, 1, 0, 0, 0},
{0, 0, 1, 1, 1, 0},
{1, 0, 0, 0, 0, 0},
{1, 1, 0, 0, 1, 0},
{1, 0, 1, 0, 0, 1},
{1, 1, 1, 1, 0, 0},
{1, 1, 1, 1, 1, 1},
};
const int opt_count = sizeof(options) / sizeof(options[0]);
for (int i = 0; i < opt_count; i++)
{
ncnn::Option opt;
opt.num_threads = 1;
opt.use_packing_layout = options[i][0];
opt.use_fp16_packed = options[i][1];
opt.use_fp16_storage = options[i][2];
opt.use_fp16_arithmetic = options[i][3];
opt.use_bf16_storage = options[i][4];
opt.use_shader_pack8 = options[i][5];
int ret = test_layer_opt(layer_type, pd, weights, opt, a, epsilon, func, flag);
if (ret != 0)
return ret;
}
return 0;
}
class TestOOMAllocator : public ncnn::UnlockedPoolAllocator
{
public:
TestOOMAllocator();
virtual void* fastMalloc(size_t size);
virtual void fastFree(void* ptr);
ncnn::Mutex lock;
int counter;
int failid;
};
TestOOMAllocator::TestOOMAllocator()
{
counter = 0;
failid = INT_MAX;
}
void* TestOOMAllocator::fastMalloc(size_t size)
{
lock.lock();
void* ptr;
if (counter == failid)
{
ptr = 0;
}
else
{
ptr = ncnn::UnlockedPoolAllocator::fastMalloc(size);
}
counter++;
lock.unlock();
return ptr;
}
void TestOOMAllocator::fastFree(void* ptr)
{
lock.lock();
ncnn::UnlockedPoolAllocator::fastFree(ptr);
lock.unlock();
}
int test_layer_oom_opt(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const std::vector<ncnn::Mat>& a, int top_blob_count, int flag)
{
int typeindex = ncnn::layer_to_index(layer_type);
if (typeindex == -1)
return -1;
ncnn::Layer* op = ncnn::create_layer_cpu(typeindex);
if (!op->support_packing && _opt.use_packing_layout)
{
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
delete op;
return 233;
}
op->load_param(pd);
if (op->one_blob_only && a.size() != 1)
{
fprintf(stderr, "layer with one_blob_only but consume multiple inputs\n");
delete op;
return -1;
}
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::Option opt = _opt;
opt.num_threads = 1;
opt.use_vulkan_compute = false;
op->create_pipeline(opt);
if (!op->support_packing && _opt.use_packing_layout)
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
std::vector<ncnn::Mat> a4(a.size());
for (size_t i = 0; i < a4.size(); i++)
{
convert_to_optimal_layout(a[i], a4[i], opt, op, flag);
}
TestOOMAllocator test_oom_allocator;
opt.blob_allocator = &test_oom_allocator;
opt.workspace_allocator = &test_oom_allocator;
std::vector<ncnn::Mat> c;
c.resize(top_blob_count);
if (op->support_inplace)
{
for (size_t i = 0; i < a4.size(); i++)
{
c[i] = a4[i].clone();
}
op->forward_inplace(c, opt);
}
else
{
op->forward(a4, c, opt);
}
for (int i = 0; i < top_blob_count; i++)
{
c[i].release();
}
const int alloc_count = test_oom_allocator.counter;
for (int i = 0; i < alloc_count; i++)
{
test_oom_allocator.counter = 0;
test_oom_allocator.failid = i;
int ret = 0;
if (op->support_inplace)
{
for (size_t i = 0; i < a4.size(); i++)
{
c[i] = a4[i].clone();
}
ret = op->forward_inplace(c, opt);
}
else
{
ret = op->forward(a4, c, opt);
}
for (int i = 0; i < top_blob_count; i++)
{
c[i].release();
}
if (ret != -100)
{
fprintf(stderr, "oom not catched %d/%d\n", i, alloc_count);
op->destroy_pipeline(opt);
delete op;
return -1;
}
}
op->destroy_pipeline(opt);
delete op;
return 0;
}
int test_layer_oom_opt(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const ncnn::Mat& a, int flag)
{
int typeindex = ncnn::layer_to_index(layer_type);
if (typeindex == -1)
return -1;
ncnn::Layer* op = ncnn::create_layer_cpu(typeindex);
if (!op->support_packing && _opt.use_packing_layout)
{
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
delete op;
return 233;
}
op->load_param(pd);
ncnn::ModelBinFromMatArray mb(weights.data());
op->load_model(mb);
ncnn::Option opt = _opt;
opt.num_threads = 1;
opt.use_vulkan_compute = false;
op->create_pipeline(opt);
if (!op->support_packing && _opt.use_packing_layout)
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
if (!op->support_bf16_storage && !op->support_fp16_storage && (_opt.use_bf16_storage || _opt.use_fp16_arithmetic))
{
op->destroy_pipeline(opt);
delete op;
return 233;
}
ncnn::Mat a4;
convert_to_optimal_layout(a, a4, opt, op, flag);
TestOOMAllocator test_oom_allocator;
opt.blob_allocator = &test_oom_allocator;
opt.workspace_allocator = &test_oom_allocator;
ncnn::Mat c;
if (op->support_inplace)
{
c = a4.clone();
op->forward_inplace(c, opt);
}
else
{
op->forward(a4, c, opt);
}
c.release();
const int alloc_count = test_oom_allocator.counter;
for (int i = 0; i < alloc_count; i++)
{
test_oom_allocator.counter = 0;
test_oom_allocator.failid = i;
int ret = 0;
if (op->support_inplace)
{
c = a4.clone();
ret = op->forward_inplace(c, opt);
}
else
{
ret = op->forward(a4, c, opt);
}
c.release();
if (ret != -100)
{
fprintf(stderr, "oom not catched %d/%d\n", i, alloc_count);
op->destroy_pipeline(opt);
delete op;
return -1;
}
}
op->destroy_pipeline(opt);
delete op;
return 0;
}
int test_layer_oom(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const std::vector<ncnn::Mat>& a, int top_blob_count, int flag)
{
// pack fp16p fp16s fp16a bf16s shader8
const int options[][6] = {
{0, 0, 0, 0, 0, 0},
{0, 0, 1, 0, 0, 0},
{0, 0, 1, 1, 1, 0},
{1, 0, 0, 0, 0, 0},
{1, 1, 0, 0, 1, 0},
{1, 0, 1, 0, 0, 1},
{1, 1, 1, 1, 0, 0},
{1, 1, 1, 1, 1, 1},
};
const int opt_count = sizeof(options) / sizeof(options[0]);
for (int i = 0; i < opt_count; i++)
{
ncnn::Option opt;
opt.num_threads = 1;
opt.use_packing_layout = options[i][0];
opt.use_fp16_packed = options[i][1];
opt.use_fp16_storage = options[i][2];
opt.use_fp16_arithmetic = options[i][3];
opt.use_bf16_storage = options[i][4];
opt.use_shader_pack8 = options[i][5];
int ret = test_layer_oom_opt(layer_type, pd, weights, opt, a, top_blob_count, flag);
if (ret != 233 && ret != 0)
return ret;
}
return 0;
}
int test_layer_oom(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Mat& a, int flag)
{
// pack fp16p fp16s fp16a bf16s shader8
const int options[][6] = {
{0, 0, 0, 0, 0, 0},
{0, 0, 1, 0, 0, 0},
{0, 0, 1, 1, 1, 0},
{1, 0, 0, 0, 0, 0},
{1, 1, 0, 0, 1, 0},
{1, 0, 1, 0, 0, 1},
{1, 1, 1, 1, 0, 0},
{1, 1, 1, 1, 1, 1},
};
const int opt_count = sizeof(options) / sizeof(options[0]);
for (int i = 0; i < opt_count; i++)
{
ncnn::Option opt;
opt.num_threads = 1;
opt.use_packing_layout = options[i][0];
opt.use_fp16_packed = options[i][1];
opt.use_fp16_storage = options[i][2];
opt.use_fp16_arithmetic = options[i][3];
opt.use_bf16_storage = options[i][4];
opt.use_shader_pack8 = options[i][5];
int ret = test_layer_oom_opt(layer_type, pd, weights, opt, a, flag);
if (ret != 233 && ret != 0)
return ret;
}
return 0;
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
C/C++
1
https://gitee.com/Tencent/ncnn.git
git@gitee.com:Tencent/ncnn.git
Tencent
ncnn
ncnn
master

搜索帮助