diff --git a/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_core.txt b/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_core.txt new file mode 100644 index 0000000000000..3a028cd380029 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_core.txt @@ -0,0 +1,301 @@ +vaba_s16 +vaba_s32 +vaba_s8 +vaba_u16 +vaba_u32 +vaba_u8 +vabaq_s16 +vabaq_s32 +vabaq_s8 +vabaq_u16 +vabaq_u32 +vabaq_u8 +vhadd_s16 +vhadd_s32 +vhadd_s8 +vhadd_u16 +vhadd_u32 +vhadd_u8 +vhaddq_s16 +vhaddq_s32 +vhaddq_s8 +vhaddq_u16 +vhaddq_u32 +vhaddq_u8 +vhsub_s16 +vhsub_s32 +vhsub_s8 +vhsub_u16 +vhsub_u32 +vhsub_u8 +vhsubq_s16 +vhsubq_s32 +vhsubq_s8 +vhsubq_u16 +vhsubq_u32 +vhsubq_u8 +vld1_f16_x2 +vld1_f16_x3 +vld1_f16_x4 +vld1_f32_x2 +vld1_f32_x3 +vld1_f32_x4 +vld1_f64_x2 +vld1_f64_x3 +vld1_f64_x4 +vld1_p16_x2 +vld1_p16_x3 +vld1_p16_x4 +vld1_p64_x2 +vld1_p64_x3 +vld1_p64_x4 +vld1_p8_x2 +vld1_p8_x3 +vld1_p8_x4 +vld1_s16_x2 +vld1_s16_x3 +vld1_s16_x4 +vld1_s32_x2 +vld1_s32_x3 +vld1_s32_x4 +vld1_s64_x2 +vld1_s64_x3 +vld1_s64_x4 +vld1_s8_x2 +vld1_s8_x3 +vld1_s8_x4 +vld1_u16_x2 +vld1_u16_x3 +vld1_u16_x4 +vld1_u32_x2 +vld1_u32_x3 +vld1_u32_x4 +vld1_u64_x2 +vld1_u64_x3 +vld1_u64_x4 +vld1_u8_x2 +vld1_u8_x3 +vld1_u8_x4 +vld1q_f16_x2 +vld1q_f16_x3 +vld1q_f16_x4 +vld1q_f32_x2 +vld1q_f32_x3 +vld1q_f32_x4 +vld1q_f64_x2 +vld1q_f64_x3 +vld1q_f64_x4 +vld1q_p16_x2 +vld1q_p16_x3 +vld1q_p16_x4 +vld1q_p64_x2 +vld1q_p64_x3 +vld1q_p64_x4 +vld1q_p8_x2 +vld1q_p8_x3 +vld1q_p8_x4 +vld1q_s16_x2 +vld1q_s16_x3 +vld1q_s16_x4 +vld1q_s32_x2 +vld1q_s32_x3 +vld1q_s32_x4 +vld1q_s64_x2 +vld1q_s64_x3 +vld1q_s64_x4 +vld1q_s8_x2 +vld1q_s8_x3 +vld1q_s8_x4 +vld1q_u16_x2 +vld1q_u16_x3 +vld1q_u16_x4 +vld1q_u32_x2 +vld1q_u32_x3 +vld1q_u32_x4 +vld1q_u64_x2 +vld1q_u64_x3 +vld1q_u64_x4 +vld1q_u8_x2 +vld1q_u8_x3 +vld1q_u8_x4 +vld2_f16 +vld2_f32 +vld2_lane_f16 +vld2_lane_f32 +vld2_lane_s16 +vld2_lane_s32 +vld2_lane_s64 +vld2_lane_s8 +vld2_lane_u16 +vld2_lane_u32 +vld2_lane_u64 +vld2_lane_u8 +vld2_p16 +vld2_p8 +vld2_s16 +vld2_s32 +vld2_s64 +vld2_s8 +vld2_u16 +vld2_u32 +vld2_u64 +vld2_u8 +vld2q_f16 +vld2q_f32 +vld2q_f64 +vld2q_lane_f16 +vld2q_lane_f32 +vld2q_lane_s16 +vld2q_lane_s32 +vld2q_lane_s64 +vld2q_lane_s8 +vld2q_lane_u16 +vld2q_lane_u32 +vld2q_lane_u64 +vld2q_lane_u8 +vld2q_p16 +vld2q_p8 +vld2q_s16 +vld2q_s32 +vld2q_s64 +vld2q_s8 +vld2q_u16 +vld2q_u32 +vld2q_u64 +vld2q_u8 +vld3_f16 +vld3_lane_f16 +vld3_lane_f32 +vld3_lane_s16 +vld3_lane_s32 +vld3_lane_s64 +vld3_lane_s8 +vld3_lane_u16 +vld3_lane_u32 +vld3_lane_u64 +vld3_lane_u8 +vld3q_f16 +vld3q_lane_f16 +vld3q_lane_f32 +vld3q_lane_s16 +vld3q_lane_s32 +vld3q_lane_s64 +vld3q_lane_s8 +vld3q_lane_u16 +vld3q_lane_u32 +vld3q_lane_u64 +vld3q_lane_u8 +vld4_f16 +vld4_lane_f16 +vld4_lane_f32 +vld4_lane_s16 +vld4_lane_s32 +vld4_lane_s64 +vld4_lane_s8 +vld4_lane_u16 +vld4_lane_u32 +vld4_lane_u64 +vld4_lane_u8 +vld4q_f16 +vld4q_lane_f16 +vld4q_lane_f32 +vld4q_lane_s16 +vld4q_lane_s32 +vld4q_lane_s64 +vld4q_lane_s8 +vld4q_lane_u16 +vld4q_lane_u32 +vld4q_lane_u64 +vld4q_lane_u8 +vqtbl2_p8 +vqtbl2_s8 +vqtbl2_u8 +vqtbl2q_p8 +vqtbl2q_s8 +vqtbl2q_u8 +vqtbl3_p8 +vqtbl3_s8 +vqtbl3_u8 +vqtbl3q_p8 +vqtbl3q_s8 +vqtbl3q_u8 +vqtbl4_p8 +vqtbl4_s8 +vqtbl4_u8 +vqtbl4q_p8 +vqtbl4q_s8 +vqtbl4q_u8 +vqtbx1_p8 +vqtbx1_s8 +vqtbx1_u8 +vqtbx1q_p8 +vqtbx1q_s8 +vqtbx1q_u8 +vqtbx2_p8 +vqtbx2_s8 +vqtbx2_u8 +vqtbx2q_p8 +vqtbx2q_s8 +vqtbx2q_u8 +vqtbx3_p8 +vqtbx3_s8 +vqtbx3_u8 +vqtbx3q_p8 +vqtbx3q_s8 +vqtbx3q_u8 +vqtbx4_p8 +vqtbx4_s8 +vqtbx4_u8 +vqtbx4q_p8 +vqtbx4q_s8 +vqtbx4q_u8 +vrhadd_s16 +vrhadd_s32 +vrhadd_s8 +vrhadd_u16 +vrhadd_u32 +vrhadd_u8 +vrhaddq_s16 +vrhaddq_s32 +vrhaddq_s8 +vrhaddq_u16 +vrhaddq_u32 +vrhaddq_u8 +vsli_n_p16 +vsli_n_p8 +vsli_n_s16 +vsli_n_s32 +vsli_n_s64 +vsli_n_s8 +vsli_n_u16 +vsli_n_u32 +vsli_n_u64 +vsli_n_u8 +vsliq_n_p16 +vsliq_n_p8 +vsliq_n_s16 +vsliq_n_s32 +vsliq_n_s64 +vsliq_n_s8 +vsliq_n_u16 +vsliq_n_u32 +vsliq_n_u64 +vsliq_n_u8 +vtbl3_p8 +vtbl3_s8 +vtbl3_u8 +vtbl4_p8 +vtbl4_s8 +vtbl4_u8 +vtbx1_p8 +vtbx1_s8 +vtbx1_u8 +vtbx2_p8 +vtbx2_s8 +vtbx2_u8 +vtbx3_p8 +vtbx3_s8 +vtbx3_u8 +vtbx4_p8 +vtbx4_s8 +vtbx4_u8 diff --git a/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_intrinsics_clang.txt b/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_intrinsics_clang.txt new file mode 100644 index 0000000000000..5f11867fd3ee9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_intrinsics_clang.txt @@ -0,0 +1 @@ +vcvth_s16_f16 diff --git a/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_intrinsics_common.txt b/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_intrinsics_common.txt new file mode 100644 index 0000000000000..a452f0c16b593 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/stdarch_skip/aarch64_intrinsics_common.txt @@ -0,0 +1,1173 @@ +__jcvt +vaba_s16 +vaba_s32 +vaba_s8 +vaba_u16 +vaba_u32 +vaba_u8 +vabal_high_s16 +vabal_high_s32 +vabal_high_s8 +vabal_high_u16 +vabal_high_u32 +vabal_high_u8 +vabal_s16 +vabal_s32 +vabal_s8 +vabal_u16 +vabal_u32 +vabal_u8 +vabaq_s16 +vabaq_s32 +vabaq_s8 +vabaq_u16 +vabaq_u32 +vabaq_u8 +vabd_f16 +vabd_f32 +vabd_f64 +vabd_s16 +vabd_s32 +vabd_s8 +vabd_u16 +vabd_u32 +vabd_u8 +vabdd_f64 +vabdh_f16 +vabdl_high_s16 +vabdl_high_s32 +vabdl_high_s8 +vabdl_high_u16 +vabdl_high_u32 +vabdl_high_u8 +vabdl_s16 +vabdl_s32 +vabdl_s8 +vabdl_u16 +vabdl_u32 +vabdl_u8 +vabdq_f16 +vabdq_f32 +vabdq_f64 +vabdq_s16 +vabdq_s32 +vabdq_s8 +vabdq_u16 +vabdq_u32 +vabdq_u8 +vabds_f32 +vabsd_s64 +vaddlv_s16 +vaddlv_s32 +vaddlv_s8 +vaddlv_u16 +vaddlv_u32 +vaddlv_u8 +vaddlvq_s16 +vaddlvq_s32 +vaddlvq_s8 +vaddlvq_u16 +vaddlvq_u32 +vaddvq_f64 +vamax_f16 +vamaxq_f16 +vamin_f16 +vaminq_f16 +vcadd_rot270_f16 +vcadd_rot270_f32 +vcadd_rot90_f16 +vcadd_rot90_f32 +vcaddq_rot270_f16 +vcaddq_rot270_f32 +vcaddq_rot270_f64 +vcaddq_rot90_f16 +vcaddq_rot90_f32 +vcaddq_rot90_f64 +vcage_f16 +vcage_f32 +vcage_f64 +vcaged_f64 +vcageh_f16 +vcageq_f16 +vcageq_f32 +vcageq_f64 +vcages_f32 +vcagt_f16 +vcagt_f32 +vcagt_f64 +vcagtd_f64 +vcagth_f16 +vcagtq_f16 +vcagtq_f32 +vcagtq_f64 +vcagts_f32 +vcale_f16 +vcale_f32 +vcale_f64 +vcaled_f64 +vcaleh_f16 +vcaleq_f16 +vcaleq_f32 +vcaleq_f64 +vcales_f32 +vcalt_f16 +vcalt_f32 +vcalt_f64 +vcaltd_f64 +vcalth_f16 +vcaltq_f16 +vcaltq_f32 +vcaltq_f64 +vcalts_f32 +vcmla_f16 +vcmla_f32 +vcmla_lane_f16 +vcmla_lane_f32 +vcmla_laneq_f16 +vcmla_laneq_f32 +vcmla_rot180_f16 +vcmla_rot180_f32 +vcmla_rot180_lane_f16 +vcmla_rot180_lane_f32 +vcmla_rot180_laneq_f16 +vcmla_rot180_laneq_f32 +vcmla_rot270_f16 +vcmla_rot270_f32 +vcmla_rot270_lane_f16 +vcmla_rot270_lane_f32 +vcmla_rot270_laneq_f16 +vcmla_rot270_laneq_f32 +vcmla_rot90_f16 +vcmla_rot90_f32 +vcmla_rot90_lane_f16 +vcmla_rot90_lane_f32 +vcmla_rot90_laneq_f16 +vcmla_rot90_laneq_f32 +vcmlaq_f16 +vcmlaq_f32 +vcmlaq_f64 +vcmlaq_lane_f16 +vcmlaq_lane_f32 +vcmlaq_laneq_f16 +vcmlaq_laneq_f32 +vcmlaq_rot180_f16 +vcmlaq_rot180_f32 +vcmlaq_rot180_f64 +vcmlaq_rot180_lane_f16 +vcmlaq_rot180_lane_f32 +vcmlaq_rot180_laneq_f16 +vcmlaq_rot180_laneq_f32 +vcmlaq_rot270_f16 +vcmlaq_rot270_f32 +vcmlaq_rot270_f64 +vcmlaq_rot270_lane_f16 +vcmlaq_rot270_lane_f32 +vcmlaq_rot270_laneq_f16 +vcmlaq_rot270_laneq_f32 +vcmlaq_rot90_f16 +vcmlaq_rot90_f32 +vcmlaq_rot90_f64 +vcmlaq_rot90_lane_f16 +vcmlaq_rot90_lane_f32 +vcmlaq_rot90_laneq_f16 +vcmlaq_rot90_laneq_f32 +vcvt_n_f16_s16 +vcvt_n_f16_u16 +vcvt_n_f32_s32 +vcvt_n_f32_u32 +vcvt_n_f64_s64 +vcvt_n_f64_u64 +vcvt_n_s16_f16 +vcvt_n_s32_f32 +vcvt_n_s64_f64 +vcvt_n_u16_f16 +vcvt_n_u32_f32 +vcvt_n_u64_f64 +vcvt_s32_f32 +vcvt_s64_f64 +vcvt_u32_f32 +vcvt_u64_f64 +vcvta_s16_f16 +vcvta_s32_f32 +vcvta_s64_f64 +vcvta_u16_f16 +vcvta_u32_f32 +vcvta_u64_f64 +vcvtad_s64_f64 +vcvtad_u64_f64 +vcvtah_s16_f16 +vcvtah_s32_f16 +vcvtah_s64_f16 +vcvtah_u16_f16 +vcvtah_u32_f16 +vcvtah_u64_f16 +vcvtaq_s16_f16 +vcvtaq_s32_f32 +vcvtaq_s64_f64 +vcvtaq_u16_f16 +vcvtaq_u32_f32 +vcvtaq_u64_f64 +vcvtas_s32_f32 +vcvtas_u32_f32 +vcvtd_n_f64_s64 +vcvtd_n_f64_u64 +vcvtd_n_s64_f64 +vcvtd_n_u64_f64 +vcvth_n_f16_s16 +vcvth_n_f16_s32 +vcvth_n_f16_s64 +vcvth_n_f16_u16 +vcvth_n_f16_u32 +vcvth_n_f16_u64 +vcvth_n_s16_f16 +vcvth_n_s32_f16 +vcvth_n_s64_f16 +vcvth_n_u16_f16 +vcvth_n_u32_f16 +vcvth_n_u64_f16 +vcvtm_s16_f16 +vcvtm_s32_f32 +vcvtm_s64_f64 +vcvtm_u16_f16 +vcvtm_u32_f32 +vcvtm_u64_f64 +vcvtmd_s64_f64 +vcvtmd_u64_f64 +vcvtmh_s16_f16 +vcvtmh_s32_f16 +vcvtmh_s64_f16 +vcvtmh_u16_f16 +vcvtmh_u32_f16 +vcvtmh_u64_f16 +vcvtmq_s16_f16 +vcvtmq_s32_f32 +vcvtmq_s64_f64 +vcvtmq_u16_f16 +vcvtmq_u32_f32 +vcvtmq_u64_f64 +vcvtms_s32_f32 +vcvtms_u32_f32 +vcvtn_s16_f16 +vcvtn_s32_f32 +vcvtn_s64_f64 +vcvtn_u16_f16 +vcvtn_u32_f32 +vcvtn_u64_f64 +vcvtnd_s64_f64 +vcvtnd_u64_f64 +vcvtnh_s16_f16 +vcvtnh_s32_f16 +vcvtnh_s64_f16 +vcvtnh_u16_f16 +vcvtnh_u32_f16 +vcvtnh_u64_f16 +vcvtnq_s16_f16 +vcvtnq_s64_f64 +vcvtnq_u16_f16 +vcvtnq_u32_f32 +vcvtnq_u64_f64 +vcvtns_s32_f32 +vcvtns_u32_f32 +vcvtp_s16_f16 +vcvtp_s32_f32 +vcvtp_s64_f64 +vcvtp_u16_f16 +vcvtp_u32_f32 +vcvtp_u64_f64 +vcvtpd_s64_f64 +vcvtpd_u64_f64 +vcvtph_s16_f16 +vcvtph_s32_f16 +vcvtph_s64_f16 +vcvtph_u16_f16 +vcvtph_u32_f16 +vcvtph_u64_f16 +vcvtpq_s16_f16 +vcvtpq_s32_f32 +vcvtpq_s64_f64 +vcvtpq_u16_f16 +vcvtpq_u32_f32 +vcvtpq_u64_f64 +vcvtps_s32_f32 +vcvtps_u32_f32 +vcvtq_n_f16_s16 +vcvtq_n_f16_u16 +vcvtq_n_f32_s32 +vcvtq_n_f32_u32 +vcvtq_n_f64_s64 +vcvtq_n_f64_u64 +vcvtq_n_s16_f16 +vcvtq_n_s32_f32 +vcvtq_n_s64_f64 +vcvtq_n_u16_f16 +vcvtq_n_u32_f32 +vcvtq_n_u64_f64 +vcvtq_s64_f64 +vcvtq_u32_f32 +vcvtq_u64_f64 +vcvts_n_f32_s32 +vcvts_n_f32_u32 +vcvts_n_s32_f32 +vcvts_n_u32_f32 +vcvtx_f32_f64 +vcvtx_high_f32_f64 +vcvtxd_f32_f64 +vdot_lane_s32 +vdot_lane_u32 +vdot_laneq_s32 +vdot_laneq_u32 +vdot_s32 +vdot_u32 +vdotq_lane_s32 +vdotq_lane_u32 +vdotq_laneq_s32 +vdotq_laneq_u32 +vdotq_s32 +vdotq_u32 +vfmlal_high_f16 +vfmlal_lane_high_f16 +vfmlal_lane_low_f16 +vfmlal_laneq_high_f16 +vfmlal_laneq_low_f16 +vfmlal_low_f16 +vfmlalq_high_f16 +vfmlalq_lane_high_f16 +vfmlalq_lane_low_f16 +vfmlalq_laneq_high_f16 +vfmlalq_laneq_low_f16 +vfmlalq_low_f16 +vfmlsl_high_f16 +vfmlsl_lane_high_f16 +vfmlsl_lane_low_f16 +vfmlsl_laneq_high_f16 +vfmlsl_laneq_low_f16 +vfmlsl_low_f16 +vfmlslq_high_f16 +vfmlslq_lane_high_f16 +vfmlslq_lane_low_f16 +vfmlslq_laneq_high_f16 +vfmlslq_laneq_low_f16 +vfmlslq_low_f16 +vhadd_s16 +vhadd_s32 +vhadd_s8 +vhadd_u16 +vhadd_u32 +vhadd_u8 +vhaddq_s16 +vhaddq_s32 +vhaddq_s8 +vhaddq_u16 +vhaddq_u32 +vhaddq_u8 +vhsub_s16 +vhsub_s32 +vhsub_s8 +vhsub_u16 +vhsub_u32 +vhsub_u8 +vhsubq_s16 +vhsubq_s32 +vhsubq_s8 +vhsubq_u16 +vhsubq_u32 +vhsubq_u8 +vld1_dup_f16 +vld1_f16 +vld1_f16_x2 +vld1_f16_x3 +vld1_f16_x4 +vld1_lane_f16 +vld1q_dup_f16 +vld1q_f16 +vld1q_f16_x2 +vld1q_f16_x3 +vld1q_f16_x4 +vld1q_lane_f16 +vld2_dup_f16 +vld2_f16 +vld2_lane_f16 +vld2q_dup_f16 +vld2q_f16 +vld2q_lane_f16 +vld3_dup_f16 +vld3_f16 +vld3_lane_f16 +vld3q_dup_f16 +vld3q_f16 +vld3q_lane_f16 +vld4_dup_f16 +vld4_f16 +vld4_lane_f16 +vld4q_dup_f16 +vld4q_f16 +vld4q_lane_f16 +vluti2_lane_f16 +vluti2_laneq_f16 +vluti2q_lane_f16 +vluti2q_laneq_f16 +vluti4q_lane_f16_x2 +vluti4q_laneq_f16_x2 +vmax_f16 +vmaxh_f16 +vmaxnm_f16 +vmaxnm_f32 +vmaxnm_f64 +vmaxnmh_f16 +vmaxnmq_f16 +vmaxnmq_f32 +vmaxnmq_f64 +vmaxnmv_f16 +vmaxnmv_f32 +vmaxnmvq_f16 +vmaxnmvq_f32 +vmaxnmvq_f64 +vmaxq_f16 +vmaxv_f16 +vmaxv_f32 +vmaxvq_f16 +vmaxvq_f32 +vmaxvq_f64 +vmin_f16 +vminh_f16 +vminnm_f16 +vminnm_f32 +vminnm_f64 +vminnmh_f16 +vminnmq_f16 +vminnmq_f32 +vminnmq_f64 +vminnmv_f16 +vminnmv_f32 +vminnmvq_f16 +vminnmvq_f32 +vminnmvq_f64 +vminq_f16 +vminv_f16 +vminv_f32 +vminvq_f16 +vminvq_f32 +vminvq_f64 +vmmlaq_s32 +vmmlaq_u32 +vmul_p8 +vmulq_p8 +vmulx_f16 +vmulx_f32 +vmulx_f64 +vmulx_lane_f16 +vmulx_lane_f32 +vmulx_lane_f64 +vmulx_laneq_f16 +vmulx_laneq_f32 +vmulx_laneq_f64 +vmulx_n_f16 +vmulxd_f64 +vmulxd_lane_f64 +vmulxd_laneq_f64 +vmulxh_f16 +vmulxh_lane_f16 +vmulxh_laneq_f16 +vmulxq_f16 +vmulxq_f32 +vmulxq_f64 +vmulxq_lane_f16 +vmulxq_lane_f32 +vmulxq_lane_f64 +vmulxq_laneq_f16 +vmulxq_laneq_f32 +vmulxq_laneq_f64 +vmulxq_n_f16 +vmulxs_f32 +vmulxs_lane_f32 +vmulxs_laneq_f32 +vpadd_f16 +vpadd_f32 +vpaddq_f16 +vpmax_f16 +vpmaxnm_f16 +vpmaxnm_f32 +vpmaxnmq_f16 +vpmaxnmq_f32 +vpmaxnmq_f64 +vpmaxnmqd_f64 +vpmaxnms_f32 +vpmaxq_f16 +vpmaxqd_f64 +vpmaxs_f32 +vpmin_f16 +vpminnm_f16 +vpminnm_f32 +vpminnmq_f16 +vpminnmq_f32 +vpminnmq_f64 +vpminnmqd_f64 +vpminnms_f32 +vpminq_f16 +vpminqd_f64 +vpmins_f32 +vqabs_s16 +vqabs_s32 +vqabs_s64 +vqabs_s8 +vqabsb_s8 +vqabsd_s64 +vqabsh_s16 +vqabsq_s16 +vqabsq_s32 +vqabsq_s64 +vqabsq_s8 +vqabss_s32 +vqaddd_s64 +vqaddd_u64 +vqadds_s32 +vqadds_u32 +vqdmlal_high_lane_s16 +vqdmlal_high_lane_s32 +vqdmlal_high_laneq_s16 +vqdmlal_high_laneq_s32 +vqdmlal_high_n_s16 +vqdmlal_high_n_s32 +vqdmlal_high_s16 +vqdmlal_high_s32 +vqdmlal_lane_s16 +vqdmlal_lane_s32 +vqdmlal_laneq_s16 +vqdmlal_laneq_s32 +vqdmlal_n_s16 +vqdmlal_n_s32 +vqdmlal_s16 +vqdmlal_s32 +vqdmlalh_lane_s16 +vqdmlalh_laneq_s16 +vqdmlalh_s16 +vqdmlals_lane_s32 +vqdmlals_laneq_s32 +vqdmlals_s32 +vqdmlsl_high_lane_s16 +vqdmlsl_high_lane_s32 +vqdmlsl_high_laneq_s16 +vqdmlsl_high_laneq_s32 +vqdmlsl_high_n_s16 +vqdmlsl_high_n_s32 +vqdmlsl_high_s16 +vqdmlsl_high_s32 +vqdmlsl_lane_s16 +vqdmlsl_lane_s32 +vqdmlsl_laneq_s16 +vqdmlsl_laneq_s32 +vqdmlsl_n_s16 +vqdmlsl_n_s32 +vqdmlsl_s16 +vqdmlsl_s32 +vqdmlslh_lane_s16 +vqdmlslh_laneq_s16 +vqdmlslh_s16 +vqdmlsls_lane_s32 +vqdmlsls_laneq_s32 +vqdmlsls_s32 +vqdmull_high_lane_s16 +vqdmull_high_lane_s32 +vqdmull_high_laneq_s16 +vqdmull_high_laneq_s32 +vqdmull_high_n_s16 +vqdmull_high_n_s32 +vqdmull_high_s16 +vqdmull_high_s32 +vqdmull_lane_s16 +vqdmull_lane_s32 +vqdmull_laneq_s16 +vqdmull_laneq_s32 +vqdmull_n_s16 +vqdmull_n_s32 +vqdmull_s16 +vqdmull_s32 +vqdmullh_lane_s16 +vqdmullh_laneq_s16 +vqdmullh_s16 +vqdmulls_lane_s32 +vqdmulls_laneq_s32 +vqdmulls_s32 +vqmovn_high_s16 +vqmovn_high_s32 +vqmovn_high_s64 +vqmovn_high_u16 +vqmovn_high_u32 +vqmovn_high_u64 +vqmovn_s16 +vqmovn_s32 +vqmovn_s64 +vqmovn_u16 +vqmovn_u32 +vqmovn_u64 +vqmovnd_s64 +vqmovnd_u64 +vqmovnh_s16 +vqmovnh_u16 +vqmovns_s32 +vqmovns_u32 +vqmovun_high_s16 +vqmovun_high_s32 +vqmovun_high_s64 +vqmovun_s16 +vqmovun_s32 +vqmovun_s64 +vqmovund_s64 +vqmovunh_s16 +vqmovuns_s32 +vqneg_s16 +vqneg_s32 +vqneg_s64 +vqneg_s8 +vqnegb_s8 +vqnegd_s64 +vqnegh_s16 +vqnegq_s16 +vqnegq_s32 +vqnegq_s64 +vqnegq_s8 +vqnegs_s32 +vqrdmlah_lane_s16 +vqrdmlah_lane_s32 +vqrdmlah_laneq_s16 +vqrdmlah_laneq_s32 +vqrdmlah_s16 +vqrdmlah_s32 +vqrdmlahh_lane_s16 +vqrdmlahh_laneq_s16 +vqrdmlahh_s16 +vqrdmlahq_lane_s16 +vqrdmlahq_lane_s32 +vqrdmlahq_laneq_s16 +vqrdmlahq_laneq_s32 +vqrdmlahq_s16 +vqrdmlahq_s32 +vqrdmlahs_lane_s32 +vqrdmlahs_laneq_s32 +vqrdmlahs_s32 +vqrdmlsh_lane_s16 +vqrdmlsh_lane_s32 +vqrdmlsh_laneq_s16 +vqrdmlsh_laneq_s32 +vqrdmlsh_s16 +vqrdmlsh_s32 +vqrdmlshh_lane_s16 +vqrdmlshh_laneq_s16 +vqrdmlshh_s16 +vqrdmlshq_lane_s16 +vqrdmlshq_lane_s32 +vqrdmlshq_laneq_s16 +vqrdmlshq_laneq_s32 +vqrdmlshq_s16 +vqrdmlshq_s32 +vqrdmlshs_lane_s32 +vqrdmlshs_laneq_s32 +vqrdmlshs_s32 +vqrdmulh_lane_s16 +vqrdmulh_lane_s32 +vqrdmulh_laneq_s16 +vqrdmulh_laneq_s32 +vqrdmulh_n_s16 +vqrdmulh_n_s32 +vqrdmulh_s16 +vqrdmulh_s32 +vqrdmulhh_lane_s16 +vqrdmulhh_laneq_s16 +vqrdmulhh_s16 +vqrdmulhq_lane_s16 +vqrdmulhq_lane_s32 +vqrdmulhq_laneq_s16 +vqrdmulhq_laneq_s32 +vqrdmulhq_n_s16 +vqrdmulhq_n_s32 +vqrdmulhq_s16 +vqrdmulhq_s32 +vqrdmulhs_lane_s32 +vqrdmulhs_laneq_s32 +vqrdmulhs_s32 +vqrshl_s16 +vqrshl_s32 +vqrshl_s64 +vqrshl_s8 +vqrshl_u16 +vqrshl_u32 +vqrshl_u64 +vqrshl_u8 +vqrshlb_s8 +vqrshlb_u8 +vqrshld_s64 +vqrshld_u64 +vqrshlh_s16 +vqrshlh_u16 +vqrshlq_s16 +vqrshlq_s32 +vqrshlq_s64 +vqrshlq_s8 +vqrshlq_u16 +vqrshlq_u32 +vqrshlq_u64 +vqrshlq_u8 +vqrshls_s32 +vqrshls_u32 +vqrshrn_high_n_s16 +vqrshrn_high_n_s32 +vqrshrn_high_n_s64 +vqrshrn_high_n_u16 +vqrshrn_high_n_u32 +vqrshrn_high_n_u64 +vqrshrn_n_s16 +vqrshrn_n_s32 +vqrshrn_n_s64 +vqrshrn_n_u16 +vqrshrn_n_u32 +vqrshrn_n_u64 +vqrshrnd_n_s64 +vqrshrnd_n_u64 +vqrshrnh_n_s16 +vqrshrnh_n_u16 +vqrshrns_n_s32 +vqrshrns_n_u32 +vqrshrun_high_n_s16 +vqrshrun_high_n_s32 +vqrshrun_high_n_s64 +vqrshrun_n_s16 +vqrshrun_n_s32 +vqrshrun_n_s64 +vqrshrund_n_s64 +vqrshrunh_n_s16 +vqrshruns_n_s32 +vqshl_n_s16 +vqshl_n_s32 +vqshl_n_s64 +vqshl_n_s8 +vqshl_n_u16 +vqshl_n_u32 +vqshl_n_u64 +vqshl_n_u8 +vqshl_s16 +vqshl_s32 +vqshl_s64 +vqshl_s8 +vqshl_u16 +vqshl_u32 +vqshl_u64 +vqshl_u8 +vqshlb_n_s8 +vqshlb_n_u8 +vqshlb_s8 +vqshlb_u8 +vqshld_n_s64 +vqshld_n_u64 +vqshld_s64 +vqshld_u64 +vqshlh_n_s16 +vqshlh_n_u16 +vqshlh_s16 +vqshlh_u16 +vqshlq_n_s16 +vqshlq_n_s32 +vqshlq_n_s64 +vqshlq_n_s8 +vqshlq_n_u16 +vqshlq_n_u32 +vqshlq_n_u64 +vqshlq_n_u8 +vqshlq_s16 +vqshlq_s32 +vqshlq_s64 +vqshlq_s8 +vqshlq_u16 +vqshlq_u32 +vqshlq_u64 +vqshlq_u8 +vqshls_n_s32 +vqshls_n_u32 +vqshls_s32 +vqshls_u32 +vqshlu_n_s16 +vqshlu_n_s32 +vqshlu_n_s64 +vqshlu_n_s8 +vqshlub_n_s8 +vqshlud_n_s64 +vqshluh_n_s16 +vqshluq_n_s16 +vqshluq_n_s32 +vqshluq_n_s64 +vqshluq_n_s8 +vqshlus_n_s32 +vqshrn_high_n_s16 +vqshrn_high_n_s32 +vqshrn_high_n_s64 +vqshrn_high_n_u16 +vqshrn_high_n_u32 +vqshrn_high_n_u64 +vqshrn_n_s16 +vqshrn_n_s32 +vqshrn_n_s64 +vqshrn_n_u16 +vqshrn_n_u32 +vqshrn_n_u64 +vqshrnd_n_s64 +vqshrnd_n_u64 +vqshrnh_n_s16 +vqshrnh_n_u16 +vqshrns_n_s32 +vqshrns_n_u32 +vqshrun_high_n_s16 +vqshrun_high_n_s32 +vqshrun_high_n_s64 +vqshrun_n_s16 +vqshrun_n_s32 +vqshrun_n_s64 +vqshrund_n_s64 +vqshrunh_n_s16 +vqshruns_n_s32 +vqsubd_s64 +vqsubd_u64 +vqsubs_s32 +vqsubs_u32 +vqtbl2_p8 +vqtbl2_s8 +vqtbl2_u8 +vqtbl2q_p8 +vqtbl2q_s8 +vqtbl2q_u8 +vqtbl3_p8 +vqtbl3_s8 +vqtbl3_u8 +vqtbl3q_p8 +vqtbl3q_s8 +vqtbl3q_u8 +vqtbl4_p8 +vqtbl4_s8 +vqtbl4_u8 +vqtbl4q_p8 +vqtbl4q_s8 +vqtbl4q_u8 +vqtbx1_p8 +vqtbx1_s8 +vqtbx1_u8 +vqtbx1q_p8 +vqtbx1q_s8 +vqtbx1q_u8 +vqtbx2_p8 +vqtbx2_s8 +vqtbx2_u8 +vqtbx2q_p8 +vqtbx2q_s8 +vqtbx2q_u8 +vqtbx3_p8 +vqtbx3_s8 +vqtbx3_u8 +vqtbx3q_p8 +vqtbx3q_s8 +vqtbx3q_u8 +vqtbx4_p8 +vqtbx4_s8 +vqtbx4_u8 +vqtbx4q_p8 +vqtbx4q_s8 +vqtbx4q_u8 +vraddhn_high_s16 +vraddhn_high_s32 +vraddhn_high_s64 +vraddhn_high_u16 +vraddhn_high_u32 +vraddhn_high_u64 +vraddhn_s16 +vraddhn_s32 +vraddhn_s64 +vraddhn_u16 +vraddhn_u32 +vraddhn_u64 +vrecpe_f16 +vrecpe_f32 +vrecpe_f64 +vrecpe_u32 +vrecped_f64 +vrecpeh_f16 +vrecpeq_f16 +vrecpeq_f64 +vrecpeq_u32 +vrecpes_f32 +vrecps_f16 +vrecps_f32 +vrecps_f64 +vrecpsd_f64 +vrecpsh_f16 +vrecpsq_f16 +vrecpsq_f64 +vrecpss_f32 +vrecpxd_f64 +vrecpxh_f16 +vrecpxs_f32 +vreinterpretq_f16_p128 +vrhadd_s16 +vrhadd_s32 +vrhadd_s8 +vrhadd_u16 +vrhadd_u32 +vrhadd_u8 +vrhaddq_s16 +vrhaddq_s32 +vrhaddq_s8 +vrhaddq_u16 +vrhaddq_u32 +vrhaddq_u8 +vrnd32x_f32 +vrnd32x_f64 +vrnd32xq_f32 +vrnd32xq_f64 +vrnd32z_f32 +vrnd32z_f64 +vrnd32zq_f32 +vrnd32zq_f64 +vrnd64x_f32 +vrnd64x_f64 +vrnd64xq_f32 +vrnd64xq_f64 +vrnd64z_f32 +vrnd64z_f64 +vrnd64zq_f32 +vrnd64zq_f64 +vrndi_f16 +vrndi_f32 +vrndi_f64 +vrndih_f16 +vrndiq_f16 +vrndiq_f32 +vrndiq_f64 +vrndn_f16 +vrndnh_f16 +vrndnq_f16 +vrndns_f32 +vrshl_s16 +vrshl_s32 +vrshl_s64 +vrshl_s8 +vrshl_u16 +vrshl_u32 +vrshl_u64 +vrshl_u8 +vrshld_s64 +vrshld_u64 +vrshlq_s16 +vrshlq_s32 +vrshlq_s64 +vrshlq_s8 +vrshlq_u16 +vrshlq_u32 +vrshlq_u64 +vrshlq_u8 +vrshr_n_s16 +vrshr_n_s32 +vrshr_n_s64 +vrshr_n_s8 +vrshr_n_u16 +vrshr_n_u32 +vrshr_n_u64 +vrshr_n_u8 +vrshrd_n_s64 +vrshrd_n_u64 +vrshrn_high_n_s16 +vrshrn_high_n_s32 +vrshrn_high_n_s64 +vrshrn_high_n_u16 +vrshrn_high_n_u32 +vrshrn_high_n_u64 +vrshrn_n_s16 +vrshrn_n_s32 +vrshrn_n_s64 +vrshrn_n_u16 +vrshrn_n_u32 +vrshrn_n_u64 +vrshrq_n_s16 +vrshrq_n_s32 +vrshrq_n_s64 +vrshrq_n_s8 +vrshrq_n_u16 +vrshrq_n_u32 +vrshrq_n_u64 +vrshrq_n_u8 +vrsqrte_f16 +vrsqrte_f32 +vrsqrte_f64 +vrsqrte_u32 +vrsqrted_f64 +vrsqrteh_f16 +vrsqrteq_f16 +vrsqrteq_f32 +vrsqrteq_f64 +vrsqrteq_u32 +vrsqrtes_f32 +vrsqrts_f16 +vrsqrts_f32 +vrsqrts_f64 +vrsqrtsd_f64 +vrsqrtsh_f16 +vrsqrtsq_f16 +vrsqrtsq_f32 +vrsqrtsq_f64 +vrsqrtss_f32 +vrsra_n_s16 +vrsra_n_s32 +vrsra_n_s64 +vrsra_n_s8 +vrsra_n_u16 +vrsra_n_u32 +vrsra_n_u64 +vrsra_n_u8 +vrsrad_n_s64 +vrsrad_n_u64 +vrsraq_n_s16 +vrsraq_n_s32 +vrsraq_n_s64 +vrsraq_n_s8 +vrsraq_n_u16 +vrsraq_n_u32 +vrsraq_n_u64 +vrsraq_n_u8 +vrsubhn_high_s16 +vrsubhn_high_s32 +vrsubhn_high_s64 +vrsubhn_high_u16 +vrsubhn_high_u32 +vrsubhn_high_u64 +vrsubhn_s16 +vrsubhn_s32 +vrsubhn_s64 +vrsubhn_u16 +vrsubhn_u32 +vrsubhn_u64 +vscale_f16 +vscaleq_f16 +vshl_s16 +vshl_s32 +vshl_s64 +vshl_s8 +vshl_u16 +vshl_u32 +vshl_u64 +vshl_u8 +vshld_s64 +vshld_u64 +vshlq_s16 +vshlq_s32 +vshlq_s64 +vshlq_s8 +vshlq_u16 +vshlq_u32 +vshlq_u64 +vshlq_u8 +vsli_n_p16 +vsli_n_p64 +vsli_n_p8 +vsli_n_s16 +vsli_n_s32 +vsli_n_s64 +vsli_n_s8 +vsli_n_u16 +vsli_n_u32 +vsli_n_u64 +vsli_n_u8 +vslid_n_s64 +vslid_n_u64 +vsliq_n_p16 +vsliq_n_p64 +vsliq_n_p8 +vsliq_n_s16 +vsliq_n_s32 +vsliq_n_s64 +vsliq_n_s8 +vsliq_n_u16 +vsliq_n_u32 +vsliq_n_u64 +vsliq_n_u8 +vsm3partw1q_u32 +vsm3partw2q_u32 +vsm3ss1q_u32 +vsm3tt1aq_u32 +vsm3tt1bq_u32 +vsm3tt2aq_u32 +vsm3tt2bq_u32 +vsm4ekeyq_u32 +vsm4eq_u32 +vsqadd_u16 +vsqadd_u32 +vsqadd_u64 +vsqadd_u8 +vsqaddb_u8 +vsqaddd_u64 +vsqaddh_u16 +vsqaddq_u16 +vsqaddq_u32 +vsqaddq_u64 +vsqaddq_u8 +vsqadds_u32 +vst1_f16 +vst1_f16_x2 +vst1_f16_x3 +vst1_f16_x4 +vst1_lane_f16 +vst1q_f16 +vst1q_f16_x2 +vst1q_f16_x3 +vst1q_f16_x4 +vst1q_lane_f16 +vst2_f16 +vst2_lane_f16 +vst2q_f16 +vst2q_lane_f16 +vst3_f16 +vst3_lane_f16 +vst3q_f16 +vst3q_lane_f16 +vst4_f16 +vst4_lane_f16 +vst4q_f16 +vst4q_lane_f16 +vsudot_lane_s32 +vsudot_laneq_s32 +vsudotq_lane_s32 +vsudotq_laneq_s32 +vtbl2_p8 +vtbl2_s8 +vtbl2_u8 +vtbl3_p8 +vtbl3_s8 +vtbl3_u8 +vtbl4_p8 +vtbl4_s8 +vtbl4_u8 +vtbx1_p8 +vtbx1_s8 +vtbx1_u8 +vtbx2_p8 +vtbx2_s8 +vtbx2_u8 +vtbx3_p8 +vtbx3_s8 +vtbx3_u8 +vtbx4_p8 +vtbx4_s8 +vtbx4_u8 +vuqadd_s16 +vuqadd_s32 +vuqadd_s64 +vuqadd_s8 +vuqaddb_s8 +vuqaddd_s64 +vuqaddh_s16 +vuqaddq_s16 +vuqaddq_s32 +vuqaddq_s64 +vuqaddq_s8 +vuqadds_s32 +vusdot_lane_s32 +vusdot_laneq_s32 +vusdot_s32 +vusdotq_lane_s32 +vusdotq_laneq_s32 +vusdotq_s32 +vusmmlaq_s32 diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs index 5ae76dc1e591e..3b9ae2608abfe 100644 --- a/compiler/rustc_codegen_cranelift/build_system/tests.rs +++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs @@ -1,4 +1,5 @@ use std::ffi::OsStr; +use std::fs; use std::path::PathBuf; use std::process::Command; @@ -108,7 +109,6 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ }), TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]), TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"), - TestCase::build_bin_and_run("aot.neon", "example/neon.rs", &[]), TestCase::build_bin_and_run("aot.gen_block_iterate", "example/gen_block_iterate.rs", &[]), TestCase::build_bin_and_run("aot.raw-dylib", "example/raw-dylib.rs", &[]), TestCase::custom("test.sysroot", &|runner| { @@ -182,6 +182,12 @@ static SYSROOT_TESTS_SRC: RelPath = RelPath::build("sysroot_tests"); static SYSROOT_TESTS: CargoProject = CargoProject::new(SYSROOT_TESTS_SRC, "sysroot_tests_target"); +static STDARCH_SRC: RelPath = RelPath::build("stdarch"); + +static STDARCH: CargoProject = CargoProject::new(STDARCH_SRC, "stdarch_target"); +static STDARCH_RUST_PROGRAMS: CargoProject = + CargoProject::new(RelPath::build("stdarch/rust_programs"), "stdarch_rust_programs_target"); + const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { RAND_REPO.patch(&runner.dirs); @@ -283,6 +289,116 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(test_cmd); } }), + TestCase::custom("test.stdarch-core", &|runner| { + let target = runner.target_compiler.target.as_str(); + let (arch, _) = target.split_once('-').unwrap(); + + // FIXME: Add support for remaining architectures. + if !["aarch64"].contains(&arch) { + eprintln!("Skipping `stdarch-core` tests: unsupported target"); + return; + } + + apply_patches( + &runner.dirs, + "stdarch", + &runner.stdlib_source.join("library/stdarch"), + &STDARCH_SRC.to_path(&runner.dirs), + ); + + STDARCH.clean(&runner.dirs); + + let skip = format!("build_system/stdarch_skip/{arch}_core.txt"); + let skip = fs::read_to_string(runner.dirs.source_dir.join(skip)).unwrap_or_default(); + + let skip_functions = skip + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .collect::>() + .join(","); + + if runner.is_native { + let mut test_cmd = STDARCH.test(&runner.target_compiler, &runner.dirs); + + test_cmd.env("TARGET", target); + test_cmd.env("STDARCH_TEST_SKIP_FUNCTION", &skip_functions); + + test_cmd.args(["-p", "core_arch", "--tests", "--", "-q"]); + spawn_and_wait(test_cmd); + } else { + eprintln!("Cross-Compiling: Not running tests"); + let mut build_cmd = STDARCH.build(&runner.target_compiler, &runner.dirs); + + build_cmd.env("TARGET", target); + + build_cmd.args(["-p", "core_arch", "--tests"]); + spawn_and_wait(build_cmd); + } + }), + TestCase::custom("test.stdarch-intrinsics", &|runner| { + let target = runner.target_compiler.target.as_str(); + let (arch, _) = target.split_once('-').unwrap(); + + // FIXME: Add support for remaining architectures. + let input = match arch { + "aarch64" => "intrinsics_data/arm_intrinsics.json", + _ => { + eprintln!("Skipping `stdarch-intrinsics` tests: unsupported target"); + return; + } + }; + + apply_patches( + &runner.dirs, + "stdarch", + &runner.stdlib_source.join("library/stdarch"), + &STDARCH_SRC.to_path(&runner.dirs), + ); + + STDARCH_RUST_PROGRAMS.clean(&runner.dirs); + + if !runner.is_native { + eprintln!("Cross-Compiling: Not running tests"); + return; + } + + let cc = if target.contains("apple") { "clang" } else { "gcc" }; + + let mut generate_cmd = STDARCH.run(&runner.target_compiler, &runner.dirs); + generate_cmd.current_dir(STDARCH.source_dir(&runner.dirs)); + generate_cmd.args([ + "-p", + "intrinsic-test", + "--", + input, + "--target", + target, + "--cc-arg-style", + cc, + "--skip", + &format!("crates/intrinsic-test/missing_{arch}_common.txt"), + "--skip", + &format!("crates/intrinsic-test/missing_{arch}_{cc}.txt"), + ]); + + for skip in [ + format!("build_system/stdarch_skip/{arch}_intrinsics_common.txt"), + format!("build_system/stdarch_skip/{arch}_intrinsics_{cc}.txt"), + ] { + let skip = runner.dirs.source_dir.join(skip); + + if skip.exists() { + generate_cmd.arg("--skip").arg(skip); + } + } + + spawn_and_wait(generate_cmd); + + let mut test_cmd = STDARCH_RUST_PROGRAMS.test(&runner.target_compiler, &runner.dirs); + test_cmd.args(["--workspace", "--tests", "--no-fail-fast", "--", "-q"]); + spawn_and_wait(test_cmd); + }), ]; pub(crate) fn run_tests( diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt index ca126f4e39e29..c1068e6257d73 100644 --- a/compiler/rustc_codegen_cranelift/config.txt +++ b/compiler/rustc_codegen_cranelift/config.txt @@ -22,7 +22,6 @@ aot.float-minmax-pass aot.powi_libcall_signature aot.issue-72793 aot.issue-59326 -aot.neon aot.gen_block_iterate aot.raw-dylib test.sysroot @@ -32,3 +31,5 @@ test.rust-random/rand test.regex test.graviola test.portable-simd +test.stdarch-core +test.stdarch-intrinsics diff --git a/compiler/rustc_codegen_cranelift/example/neon.rs b/compiler/rustc_codegen_cranelift/example/neon.rs deleted file mode 100644 index ba63333daa339..0000000000000 --- a/compiler/rustc_codegen_cranelift/example/neon.rs +++ /dev/null @@ -1,963 +0,0 @@ -// Most of these tests are copied from https://github.com/japaric/stdsimd/blob/0f4413d01c4f0c3ffbc5a69e9a37fbc7235b31a9/coresimd/arm/neon.rs - -#![cfg_attr(target_arch = "aarch64", feature(portable_simd))] - -#[cfg(target_arch = "aarch64")] -use std::arch::aarch64::*; -#[cfg(target_arch = "aarch64")] -use std::mem::transmute; -#[cfg(target_arch = "aarch64")] -use std::simd::*; - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "crc")] -unsafe fn test_crc32() { - assert!(std::arch::is_aarch64_feature_detected!("crc")); - - let a: u32 = 42; - let b: u64 = 0xdeadbeef; - - assert_eq!(__crc32b(a, b as u8), 0xEB0E363F); - assert_eq!(__crc32h(a, b as u16), 0x9A54BD80); - assert_eq!(__crc32w(a, b as u32), 0xF491F059); - assert_eq!(__crc32d(a, b as u64), 0xD14BBEA6); - - assert_eq!(__crc32cb(a, b as u8), 0xF67C32D8); - assert_eq!(__crc32ch(a, b as u16), 0x479108B8); - assert_eq!(__crc32cw(a, b as u32), 0x979F49F8); - assert_eq!(__crc32cd(a, b as u64), 0x0E6BE593); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_s8() { - let a = i8x8::from([1, -2, 3, -4, 5, 6, 7, 8]); - let b = i8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); - let e = i8x8::from([-2, -4, 5, 7, 0, 2, 4, 6]); - let r: i8x8 = unsafe { transmute(vpmin_s8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_s16() { - let a = i16x4::from([1, 2, 3, -4]); - let b = i16x4::from([0, 3, 2, 5]); - let e = i16x4::from([1, -4, 0, 2]); - let r: i16x4 = unsafe { transmute(vpmin_s16(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_s32() { - let a = i32x2::from([1, -2]); - let b = i32x2::from([0, 3]); - let e = i32x2::from([-2, 0]); - let r: i32x2 = unsafe { transmute(vpmin_s32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_u8() { - let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); - let b = u8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); - let e = u8x8::from([1, 3, 5, 7, 0, 2, 4, 6]); - let r: u8x8 = unsafe { transmute(vpmin_u8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_u16() { - let a = u16x4::from([1, 2, 3, 4]); - let b = u16x4::from([0, 3, 2, 5]); - let e = u16x4::from([1, 3, 0, 2]); - let r: u16x4 = unsafe { transmute(vpmin_u16(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_u32() { - let a = u32x2::from([1, 2]); - let b = u32x2::from([0, 3]); - let e = u32x2::from([1, 0]); - let r: u32x2 = unsafe { transmute(vpmin_u32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmin_f32() { - let a = f32x2::from([1., -2.]); - let b = f32x2::from([0., 3.]); - let e = f32x2::from([-2., 0.]); - let r: f32x2 = unsafe { transmute(vpmin_f32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_s8() { - let a = i8x8::from([1, -2, 3, -4, 5, 6, 7, 8]); - let b = i8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); - let e = i8x8::from([1, 3, 6, 8, 3, 5, 7, 9]); - let r: i8x8 = unsafe { transmute(vpmax_s8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_s16() { - let a = i16x4::from([1, 2, 3, -4]); - let b = i16x4::from([0, 3, 2, 5]); - let e = i16x4::from([2, 3, 3, 5]); - let r: i16x4 = unsafe { transmute(vpmax_s16(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_s32() { - let a = i32x2::from([1, -2]); - let b = i32x2::from([0, 3]); - let e = i32x2::from([1, 3]); - let r: i32x2 = unsafe { transmute(vpmax_s32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_u8() { - let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); - let b = u8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); - let e = u8x8::from([2, 4, 6, 8, 3, 5, 7, 9]); - let r: u8x8 = unsafe { transmute(vpmax_u8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_u16() { - let a = u16x4::from([1, 2, 3, 4]); - let b = u16x4::from([0, 3, 2, 5]); - let e = u16x4::from([2, 4, 3, 5]); - let r: u16x4 = unsafe { transmute(vpmax_u16(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_u32() { - let a = u32x2::from([1, 2]); - let b = u32x2::from([0, 3]); - let e = u32x2::from([2, 3]); - let r: u32x2 = unsafe { transmute(vpmax_u32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpmax_f32() { - let a = f32x2::from([1., -2.]); - let b = f32x2::from([0., 3.]); - let e = f32x2::from([1., 3.]); - let r: f32x2 = unsafe { transmute(vpmax_f32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpadd_s16() { - let a = i16x4::from([1, 2, 3, 4]); - let b = i16x4::from([0, -1, -2, -3]); - let r: i16x4 = unsafe { transmute(vpadd_s16(transmute(a), transmute(b))) }; - let e = i16x4::from([3, 7, -1, -5]); - assert_eq!(r, e); -} -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpadd_s32() { - let a = i32x2::from([1, 2]); - let b = i32x2::from([0, -1]); - let r: i32x2 = unsafe { transmute(vpadd_s32(transmute(a), transmute(b))) }; - let e = i32x2::from([3, -1]); - assert_eq!(r, e); -} -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpadd_s8() { - let a = i8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); - let b = i8x8::from([0, -1, -2, -3, -4, -5, -6, -7]); - let r: i8x8 = unsafe { transmute(vpadd_s8(transmute(a), transmute(b))) }; - let e = i8x8::from([3, 7, 11, 15, -1, -5, -9, -13]); - assert_eq!(r, e); -} -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpadd_u16() { - let a = u16x4::from([1, 2, 3, 4]); - let b = u16x4::from([30, 31, 32, 33]); - let r: u16x4 = unsafe { transmute(vpadd_u16(transmute(a), transmute(b))) }; - let e = u16x4::from([3, 7, 61, 65]); - assert_eq!(r, e); -} -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpadd_u32() { - let a = u32x2::from([1, 2]); - let b = u32x2::from([30, 31]); - let r: u32x2 = unsafe { transmute(vpadd_u32(transmute(a), transmute(b))) }; - let e = u32x2::from([3, 61]); - assert_eq!(r, e); -} -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpadd_u8() { - let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); - let b = u8x8::from([30, 31, 32, 33, 34, 35, 36, 37]); - let r: u8x8 = unsafe { transmute(vpadd_u8(transmute(a), transmute(b))) }; - let e = u8x8::from([3, 7, 11, 15, 61, 65, 69, 73]); - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vqsub_u8() { - let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 0xff]); - let b = u8x8::from([30, 1, 1, 1, 34, 0xff, 36, 37]); - let r: u8x8 = unsafe { transmute(vqsub_u8(transmute(a), transmute(b))) }; - let e = u8x8::from([0, 1, 2, 3, 0, 0, 0, 218]); - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vqadd_u8() { - let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 0xff]); - let b = u8x8::from([30, 1, 1, 1, 34, 0xff, 36, 37]); - let r: u8x8 = unsafe { transmute(vqadd_u8(transmute(a), transmute(b))) }; - let e = u8x8::from([31, 3, 4, 5, 39, 0xff, 43, 0xff]); - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vmaxq_f32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.fmax.v4f32 - let a = f32x4::from([0., -1., 2., -3.]); - let b = f32x4::from([-4., 5., -6., 7.]); - let e = f32x4::from([0., 5., 2., 7.]); - let r: f32x4 = unsafe { transmute(vmaxq_f32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vminq_f32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.fmin.v4f32 - let a = f32x4::from([0., -1., 2., -3.]); - let b = f32x4::from([-4., 5., -6., 7.]); - let e = f32x4::from([-4., -1., -6., -3.]); - let r: f32x4 = unsafe { transmute(vminq_f32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vaddvq_f32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.faddv.f32.v4f32 - let a = f32x4::from([0., 1., 2., 3.]); - let e = 6f32; - let r = unsafe { vaddvq_f32(transmute(a)) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vrndnq_f32() { - // llvm intrinsic: llvm.roundeven.v4f32 - let a = f32x4::from([0.1, -1.9, 4.5, 5.5]); - let e = f32x4::from([0., -2., 4., 6.]); - let r: f32x4 = unsafe { transmute(vrndnq_f32(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "aes")] -unsafe fn test_vaeseq_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.aese - let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let e = u8x16::from([ - 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, - 0xca, - ]); - let r: u8x16 = unsafe { transmute(vaeseq_u8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "aes")] -unsafe fn test_vaesdq_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesd - let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let e = u8x16::from([ - 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, - 0x7c, - ]); - let r: u8x16 = unsafe { transmute(vaesdq_u8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "aes")] -unsafe fn test_vaesmcq_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesmc - let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let e = u8x16::from([2, 7, 0, 5, 6, 3, 4, 1, 10, 15, 8, 13, 14, 11, 12, 9]); - let r: u8x16 = unsafe { transmute(vaesmcq_u8(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "aes")] -unsafe fn test_vaesimcq_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesimc - let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let e = u8x16::from([10, 15, 8, 13, 14, 11, 12, 9, 2, 7, 0, 5, 6, 3, 4, 1]); - let r: u8x16 = unsafe { transmute(vaesimcq_u8(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha1cq_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1c - let a = u32x4::from([0, 1, 2, 3]); - let b = 8; - let c = u32x4::from([4, 5, 6, 7]); - let e = u32x4::from([0x40072911, 0x40003948, 0x80000072, 0x80000003]); - let r: u32x4 = unsafe { transmute(vsha1cq_u32(transmute(a), b, transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -fn test_vsha1h_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1h - let a = 8; - let e = 0x00000002; - let r = vsha1h_u32(a); - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha1mq_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1m - let a = u32x4::from([0, 1, 2, 3]); - let b = 8; - let c = u32x4::from([4, 5, 6, 7]); - let e = u32x4::from([0x4007a107, 0x00003d08, 0x0000007a, 0xc0000003]); - let r: u32x4 = unsafe { transmute(vsha1mq_u32(transmute(a), b, transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha1pq_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1p - let a = u32x4::from([0, 1, 2, 3]); - let b = 8; - let c = u32x4::from([4, 5, 6, 7]); - let e = u32x4::from([0x80062d18, 0x4000315c, 0x90000062, 0x00000003]); - let r: u32x4 = unsafe { transmute(vsha1pq_u32(transmute(a), b, transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha1su0q_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1su0 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let c = u32x4::from([8, 9, 10, 11]); - let e = u32x4::from([0x0000000a, 0x0000000b, 0x0000000c, 0x0000000d]); - let r: u32x4 = unsafe { transmute(vsha1su0q_u32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha1su1q_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1su1 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let e = u32x4::from([0x0000000a, 0x0000000e, 0x0000000a, 0x00000012]); - let r: u32x4 = unsafe { transmute(vsha1su1q_u32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha256hq_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256h - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let c = u32x4::from([8, 9, 10, 11]); - let e = u32x4::from([0x27bb4ae0, 0xd8f61f7c, 0xb7c1ecdc, 0x10800215]); - let r: u32x4 = unsafe { transmute(vsha256hq_u32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha256h2q_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256h2 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let c = u32x4::from([8, 9, 10, 11]); - let e = u32x4::from([0x6989ee0d, 0x4b055920, 0x52800a12, 0x00000014]); - let r: u32x4 = unsafe { transmute(vsha256h2q_u32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha256su0q_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256su0 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let e = u32x4::from([0x02004000, 0x04008001, 0x0600c002, 0x08010003]); - let r: u32x4 = unsafe { transmute(vsha256su0q_u32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha2")] -unsafe fn test_vsha256su1q_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256su1 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let c = u32x4::from([8, 9, 10, 11]); - let e = u32x4::from([0x00044005, 0x0004e007, 0xa802211b, 0xec036145]); - let r: u32x4 = unsafe { transmute(vsha256su1q_u32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vsha512hq_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512h - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let c = u64x2::from([4, 5]); - let e = u64x2::from([0x001c805053800005, 0x0015400002800003]); - let r: u64x2 = unsafe { transmute(vsha512hq_u64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vsha512h2q_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512h2 - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let c = u64x2::from([4, 5]); - let e = u64x2::from([0x401000514a000405, 0x0000004108000005]); - let r: u64x2 = unsafe { transmute(vsha512h2q_u64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vsha512su0q_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512su0 - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let e = u64x2::from([0x8100000000000000, 0x0200000000000002]); - let r: u64x2 = unsafe { transmute(vsha512su0q_u64(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vsha512su1q_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512su1 - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let c = u64x2::from([4, 5]); - let e = u64x2::from([0x0000400000000014, 0x000060000000001e]); - let r: u64x2 = unsafe { transmute(vsha512su1q_u64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_s8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v16i8 - let a = i8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let b = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let c = i8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); - let e = i8x16::from([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]); - let r: i8x16 = unsafe { transmute(veor3q_s8(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_s16() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v8i16 - let a = i16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); - let b = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let c = i16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); - let e = i16x8::from([24, 25, 26, 27, 28, 29, 30, 31]); - let r: i16x8 = unsafe { transmute(veor3q_s16(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_s32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v4i32 - let a = i32x4::from([0, 1, 2, 3]); - let b = i32x4::from([4, 5, 6, 7]); - let c = i32x4::from([8, 9, 10, 11]); - let e = i32x4::from([12, 13, 14, 15]); - let r: i32x4 = unsafe { transmute(veor3q_s32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_s64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v2i64 - let a = i64x2::from([0, 1]); - let b = i64x2::from([2, 3]); - let c = i64x2::from([4, 5]); - let e = i64x2::from([6, 7]); - let r: i64x2 = unsafe { transmute(veor3q_s64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v16i8 - let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let c = u8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); - let e = u8x16::from([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]); - let r: u8x16 = unsafe { transmute(veor3q_u8(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_u16() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v8i16 - let a = u16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); - let b = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let c = u16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); - let e = u16x8::from([24, 25, 26, 27, 28, 29, 30, 31]); - let r: u16x8 = unsafe { transmute(veor3q_u16(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v4i32 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let c = u32x4::from([8, 9, 10, 11]); - let e = u32x4::from([12, 13, 14, 15]); - let r: u32x4 = unsafe { transmute(veor3q_u32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_veor3q_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v2i64 - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let c = u64x2::from([4, 5]); - let e = u64x2::from([6, 7]); - let r: u64x2 = unsafe { transmute(veor3q_u64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_s8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v16i8 - let a = i8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let b = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let c = i8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); - let e = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let r: i8x16 = unsafe { transmute(vbcaxq_s8(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_s16() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v8i16 - let a = i16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); - let b = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let c = i16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); - let e = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let r: i16x8 = unsafe { transmute(vbcaxq_s16(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_s32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v4i32 - let a = i32x4::from([0, 1, 2, 3]); - let b = i32x4::from([4, 5, 6, 7]); - let c = i32x4::from([8, 9, 10, 11]); - let e = i32x4::from([4, 5, 6, 7]); - let r: i32x4 = unsafe { transmute(vbcaxq_s32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_s64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v2i64 - let a = i64x2::from([0, 1]); - let b = i64x2::from([2, 3]); - let c = i64x2::from([4, 5]); - let e = i64x2::from([2, 3]); - let r: i64x2 = unsafe { transmute(vbcaxq_s64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v16i8 - let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let c = u8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); - let e = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - let r: u8x16 = unsafe { transmute(vbcaxq_u8(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_u16() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v8i16 - let a = u16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); - let b = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let c = u16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); - let e = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let r: u16x8 = unsafe { transmute(vbcaxq_u16(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v4i32 - let a = u32x4::from([0, 1, 2, 3]); - let b = u32x4::from([4, 5, 6, 7]); - let c = u32x4::from([8, 9, 10, 11]); - let e = u32x4::from([4, 5, 6, 7]); - let r: u32x4 = unsafe { transmute(vbcaxq_u32(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vbcaxq_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v2i64 - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let c = u64x2::from([4, 5]); - let e = u64x2::from([2, 3]); - let r: u64x2 = unsafe { transmute(vbcaxq_u64(transmute(a), transmute(b), transmute(c))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vrax1q_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.rax1 - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let e = u64x2::from([4, 7]); - let r: u64x2 = unsafe { transmute(vrax1q_u64(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "sha3")] -unsafe fn test_vxarq_u64() { - // AArch64 llvm intrinsic: llvm.aarch64.crypto.xar - let a = u64x2::from([0, 1]); - let b = u64x2::from([2, 3]); - let e = u64x2::from([4, 4]); - let r: u64x2 = unsafe { transmute(vxarq_u64::<63>(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "aes")] -fn test_vmull_p64() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull64 - let a: u64 = 3; - let b: u64 = 6; - let e: u128 = 10; - let r: u128 = vmull_p64(a, b); - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vmull_p8() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull.v8i16 - let a = u8x8::from([0, 1, 2, 3, 4, 5, 6, 7]); - let b = u8x8::from([8, 9, 10, 11, 12, 13, 14, 15]); - let e = u16x8::from([0x0000, 0x0009, 0x0014, 0x001d, 0x0030, 0x0039, 0x0024, 0x002d]); - let r: u16x8 = unsafe { transmute(vmull_p8(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vqdmulh_s16() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v4i16 - let a = i16x4::from([1, 2, 4, 8]); - let b = i16x4::from([16384, 16384, 16384, 16384]); - let e = i16x4::from([0, 1, 2, 4]); - let r: i16x4 = unsafe { transmute(vqdmulh_s16(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vqdmulh_s32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v2i32 - let a = i32x2::from([1, 2]); - let b = i32x2::from([1073741824, 1073741824]); - let e = i32x2::from([0, 1]); - let r: i32x2 = unsafe { transmute(vqdmulh_s32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vqdmulhq_s16() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v8i16 - let a = i16x8::from([1, 2, 4, 8, 16, 32, 64, 128]); - let b = i16x8::from([16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]); - let e = i16x8::from([0, 1, 2, 4, 8, 16, 32, 64]); - let r: i16x8 = unsafe { transmute(vqdmulhq_s16(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vqdmulhq_s32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v4i32 - let a = i32x4::from([1, 2, 4, 8]); - let b = i32x4::from([1073741824, 1073741824, 1073741824, 1073741824]); - let e = i32x4::from([0, 1, 2, 4]); - let r: i32x4 = unsafe { transmute(vqdmulhq_s32(transmute(a), transmute(b))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddl_s8() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v4i16.v8i8 - let a = i8x8::from([1, 2, 3, 4, -5, -6, -7, -8]); - let e = i16x4::from([3, 7, -11, -15]); - let r: i16x4 = unsafe { transmute(vpaddl_s8(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddl_s16() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v2i32.v4i16 - let a = i16x4::from([1, 2, -3, -4]); - let e = i32x2::from([3, -7]); - let r: i32x2 = unsafe { transmute(vpaddl_s16(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddl_s32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v1i64.v2i32 - let a = i32x2::from([1, -2]); - let e = i64x1::from([-1]); - let r: i64x1 = unsafe { transmute(vpaddl_s32(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddlq_s8() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v8i16.v16i8 - let a = i8x16::from([1, 2, 3, 4, 5, 6, 7, 8, -9, -10, -11, -12, -13, -14, -15, -16]); - let e = i16x8::from([3, 7, 11, 15, -19, -23, -27, -31]); - let r: i16x8 = unsafe { transmute(vpaddlq_s8(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddlq_s16() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v4i32.v8i16 - let a = i16x8::from([1, 2, 3, 4, -5, -6, -7, -8]); - let e = i32x4::from([3, 7, -11, -15]); - let r: i32x4 = unsafe { transmute(vpaddlq_s16(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddlq_s32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v2i64.v4i32 - let a = i32x4::from([1, 2, -3, -4]); - let e = i64x2::from([3, -7]); - let r: i64x2 = unsafe { transmute(vpaddlq_s32(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddl_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v4i16.v8i8 - let a = u8x8::from([255, 254, 253, 252, 251, 250, 249, 248]); - let e = u16x4::from([509, 505, 501, 497]); - let r: u16x4 = unsafe { transmute(vpaddl_u8(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddl_u16() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v2i32.v4i16 - let a = u16x4::from([65535, 65534, 65533, 65532]); - let e = u32x2::from([131069, 131065]); - let r: u32x2 = unsafe { transmute(vpaddl_u16(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddl_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v1i64.v2i32 - let a = u32x2::from([4294967295, 4294967294]); - let e = u64x1::from([8589934589]); - let r: u64x1 = unsafe { transmute(vpaddl_u32(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddlq_u8() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v8i16.v16i8 - let a = u8x16::from([ - 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, - ]); - let e = u16x8::from([509, 505, 501, 497, 493, 489, 485, 481]); - let r: u16x8 = unsafe { transmute(vpaddlq_u8(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddlq_u16() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v4i32.v8i16 - let a = u16x8::from([65535, 65534, 65533, 65532, 65531, 65530, 65529, 65528]); - let e = u32x4::from([131069, 131065, 131061, 131057]); - let r: u32x4 = unsafe { transmute(vpaddlq_u16(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -unsafe fn test_vpaddlq_u32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v2i64.v4i32 - let a = u32x4::from([4294967295, 4294967294, 4294967293, 4294967292]); - let e = u64x2::from([8589934589, 8589934585]); - let r: u64x2 = unsafe { transmute(vpaddlq_u32(transmute(a))) }; - assert_eq!(r, e); -} - -#[cfg(target_arch = "aarch64")] -fn main() { - unsafe { - test_vpmin_s8(); - test_vpmin_s16(); - test_vpmin_s32(); - test_vpmin_u8(); - test_vpmin_u16(); - test_vpmin_u32(); - test_vpmin_f32(); - test_vpmax_s8(); - test_vpmax_s16(); - test_vpmax_s32(); - test_vpmax_u8(); - test_vpmax_u16(); - test_vpmax_u32(); - test_vpmax_f32(); - - test_vpadd_s16(); - test_vpadd_s32(); - test_vpadd_s8(); - test_vpadd_u16(); - test_vpadd_u32(); - test_vpadd_u8(); - - test_vqsub_u8(); - test_vqadd_u8(); - - test_vmaxq_f32(); - test_vminq_f32(); - test_vaddvq_f32(); - test_vrndnq_f32(); - - test_crc32(); - - test_vaeseq_u8(); - test_vaesdq_u8(); - test_vaesmcq_u8(); - test_vaesimcq_u8(); - - test_vsha1cq_u32(); - test_vsha1h_u32(); - test_vsha1mq_u32(); - test_vsha1pq_u32(); - test_vsha1su0q_u32(); - test_vsha1su1q_u32(); - - test_vsha256hq_u32(); - test_vsha256h2q_u32(); - test_vsha256su0q_u32(); - test_vsha256su1q_u32(); - - if std::arch::is_aarch64_feature_detected!("sha3") { - test_vsha512hq_u64(); - test_vsha512h2q_u64(); - test_vsha512su0q_u64(); - test_vsha512su1q_u64(); - - test_veor3q_s8(); - test_veor3q_s16(); - test_veor3q_s32(); - test_veor3q_s64(); - test_veor3q_u8(); - test_veor3q_u16(); - test_veor3q_u32(); - test_veor3q_u64(); - - test_vbcaxq_s8(); - test_vbcaxq_s16(); - test_vbcaxq_s32(); - test_vbcaxq_s64(); - test_vbcaxq_u8(); - test_vbcaxq_u16(); - test_vbcaxq_u32(); - test_vbcaxq_u64(); - - test_vrax1q_u64(); - test_vxarq_u64(); - } - - test_vmull_p64(); - test_vmull_p8(); - - test_vqdmulh_s16(); - test_vqdmulh_s32(); - test_vqdmulhq_s16(); - test_vqdmulhq_s32(); - - test_vpaddl_s8(); - test_vpaddl_s16(); - test_vpaddl_s32(); - test_vpaddlq_s8(); - test_vpaddlq_s16(); - test_vpaddlq_s32(); - - test_vpaddl_u8(); - test_vpaddl_u16(); - test_vpaddl_u32(); - test_vpaddlq_u8(); - test_vpaddlq_u16(); - test_vpaddlq_u32(); - } -} - -#[cfg(not(target_arch = "aarch64"))] -fn main() {} diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index e1287683d495d..5fc5f17682cc8 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -294,6 +294,19 @@ unsafe fn test_simd() { if is_x86_feature_detected!("avx512f") { test_zmm_roundtrip(); } + + #[cfg(not(jit))] + { + if is_x86_feature_detected!("pclmulqdq") { + test_mm_clmulepi64_si128(); + } + if is_x86_feature_detected!("vpclmulqdq") { + test_mm256_clmulepi64_epi128(); + } + if is_x86_feature_detected!("vpclmulqdq") && is_x86_feature_detected!("avx512f") { + test_mm512_clmulepi64_epi128(); + } + } } } @@ -395,6 +408,14 @@ pub fn assert_eq_m256i(a: __m256i, b: __m256i) { } } +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx512f")] +pub fn assert_eq_m512i(a: __m512i, b: __m512i) { + unsafe { + assert_eq!(std::mem::transmute::<_, [u64; 8]>(a), std::mem::transmute::<_, [u64; 8]>(b)) + } +} + #[cfg(target_arch = "x86_64")] #[target_feature(enable = "sse2")] unsafe fn test_mm_cvtsi128_si64() { @@ -537,6 +558,62 @@ unsafe fn test_mm256_permutevar8x32_epi32() { assert_eq_m256i(r, e); } +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "pclmulqdq")] +#[cfg(not(jit))] +unsafe fn test_mm_clmulepi64_si128() { + // Constants taken from https://software.intel.com/sites/default/files/managed/72/cc/clmul-wp-rev-2.02-2014-04-20.pdf + let a = _mm_set_epi64x(0x7b5b546573745665, 0x63746f725d53475d); + let b = _mm_set_epi64x(0x4869285368617929, 0x5b477565726f6e5d); + let r00 = _mm_set_epi64x(0x1d4d84c85c3440c0, 0x929633d5d36f0451u64.cast_signed()); + let r01 = _mm_set_epi64x(0x1bd17c8d556ab5a1, 0x7fa540ac2a281315); + let r10 = _mm_set_epi64x(0x1a2bf6db3a30862f, 0xbabf262df4b7d5c9u64.cast_signed()); + let r11 = _mm_set_epi64x(0x1d1e1f2c592e7c45, 0xd66ee03e410fd4edu64.cast_signed()); + + assert_eq_m128i(_mm_clmulepi64_si128::<0x00>(a, b), r00); + assert_eq_m128i(_mm_clmulepi64_si128::<0x10>(a, b), r01); + assert_eq_m128i(_mm_clmulepi64_si128::<0x01>(a, b), r10); + assert_eq_m128i(_mm_clmulepi64_si128::<0x11>(a, b), r11); + + let a0 = _mm_set_epi64x(0x0000000000000000, 0x8000000000000000u64.cast_signed()); + let r = _mm_set_epi64x(0x4000000000000000, 0x0000000000000000); + assert_eq_m128i(_mm_clmulepi64_si128::<0x00>(a0, a0), r); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "vpclmulqdq")] +#[cfg(not(jit))] +unsafe fn test_mm256_clmulepi64_epi128() { + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let b = _mm256_setr_epi64x(5, 6, 7, 8); + + assert_eq_m256i(_mm256_clmulepi64_epi128::<0x00>(a, b), _mm256_setr_epi64x(5, 0, 9, 0)); + assert_eq_m256i(_mm256_clmulepi64_epi128::<0x01>(a, b), _mm256_setr_epi64x(10, 0, 28, 0)); + assert_eq_m256i(_mm256_clmulepi64_epi128::<0x10>(a, b), _mm256_setr_epi64x(6, 0, 24, 0)); + assert_eq_m256i(_mm256_clmulepi64_epi128::<0x11>(a, b), _mm256_setr_epi64x(12, 0, 32, 0)); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "vpclmulqdq,avx512f")] +#[cfg(not(jit))] +unsafe fn test_mm512_clmulepi64_epi128() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_setr_epi64(9, 10, 11, 12, 13, 14, 15, 16); + + assert_eq_m512i( + _mm512_clmulepi64_epi128::<0x00>(a, b), + _mm512_setr_epi64(9, 0, 29, 0, 57, 0, 45, 0), + ); + assert_eq_m512i( + _mm512_clmulepi64_epi128::<0x10>(a, b), + _mm512_setr_epi64(10, 0, 20, 0, 54, 0, 112, 0), + ); + assert_eq_m512i( + _mm512_clmulepi64_epi128::<0x11>(a, b), + _mm512_setr_epi64(20, 0, 48, 0, 36, 0, 128, 0), + ); +} + #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] #[cfg(not(jit))] diff --git a/compiler/rustc_codegen_cranelift/josh-sync.toml b/compiler/rustc_codegen_cranelift/josh-sync.toml index 2de9f9489cce3..d8b49ebac131d 100644 --- a/compiler/rustc_codegen_cranelift/josh-sync.toml +++ b/compiler/rustc_codegen_cranelift/josh-sync.toml @@ -1,3 +1,4 @@ repo = "rustc_codegen_cranelift" filter = ":~(history=\"keep-trivial-merges,no-splice\")[:rev(<=5e120485964f4857f1ad70f7d661fd244d087668:prefix=compiler/rustc_codegen_cranelift,<=7bd21608dfab11ea536f7be8936cc7dfac5864fb:SQUASH)]:/compiler/rustc_codegen_cranelift" filter-version = 2 +pull-mode = "Nightly" diff --git a/compiler/rustc_codegen_cranelift/patches/0001-stdarch-Disable-SVE.patch b/compiler/rustc_codegen_cranelift/patches/0001-stdarch-Disable-SVE.patch new file mode 100644 index 0000000000000..e7038699bc998 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0001-stdarch-Disable-SVE.patch @@ -0,0 +1,51 @@ +From e90274ba5bc4bee03a24e8c134ea223497c3a46b Mon Sep 17 00:00:00 2001 +From: Cathal Mullan +Date: Sat, 13 Jun 2026 20:32:02 +0100 +Subject: [PATCH] Disable SVE + +Cranelift doesn't support scalable vectors. +--- + crates/core_arch/src/aarch64/mod.rs | 8 ++++---- + crates/intrinsic-test/src/arm/mod.rs | 1 + + 2 files changed, 5 insertions(+), 4 deletions(-) + +diff --git a/crates/core_arch/src/aarch64/mod.rs b/crates/core_arch/src/aarch64/mod.rs +index 34fa5c8..a870e31 100644 +--- a/crates/core_arch/src/aarch64/mod.rs ++++ b/crates/core_arch/src/aarch64/mod.rs +@@ -19,16 +19,16 @@ mod neon; + pub use self::neon::*; + + // The rest of `core_arch::aarch64` is available on `arm64ec` but SVE is not supported on `arm64ec`. +-#[cfg(any(all(target_arch = "aarch64", target_endian = "little"), doc))] ++#[cfg(all(not(test), any(all(target_arch = "aarch64", target_endian = "little"), doc)))] + mod sve; +-#[cfg(any(all(target_arch = "aarch64", target_endian = "little"), doc))] ++#[cfg(all(not(test), any(all(target_arch = "aarch64", target_endian = "little"), doc)))] + #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] + pub use self::sve::*; + + // The rest of `core_arch::aarch64` is available on `arm64ec` but SVE is not supported on `arm64ec`. +-#[cfg(any(all(target_arch = "aarch64", target_endian = "little"), doc))] ++#[cfg(all(not(test), any(all(target_arch = "aarch64", target_endian = "little"), doc)))] + mod sve2; +-#[cfg(any(all(target_arch = "aarch64", target_endian = "little"), doc))] ++#[cfg(all(not(test), any(all(target_arch = "aarch64", target_endian = "little"), doc)))] + #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] + pub use self::sve2::*; + +diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs +index 9c2d0f3..3825f41 100644 +--- a/crates/intrinsic-test/src/arm/mod.rs ++++ b/crates/intrinsic-test/src/arm/mod.rs +@@ -66,6 +66,7 @@ impl SupportedArchitecture for Arm { + + let intrinsics = intrinsics + .into_iter() ++ .filter(|i| i.extension != "SVE" && i.extension != "SVE2") + // Skip intrinsics that don't return a value. + .filter(|i| i.results.kind() != TypeKind::Void) + // Skip bfloat intrinsics - not currently supported +-- +2.54.0 + diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain.toml b/compiler/rustc_codegen_cranelift/rust-toolchain.toml index d7e313a7a33ed..a6468283e1270 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain.toml +++ b/compiler/rustc_codegen_cranelift/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-09-11" +channel = "nightly-2026-09-30" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh index e26be307328f0..8bd41f98ce550 100755 --- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh +++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh @@ -2,75 +2,34 @@ set -e -TOOLCHAIN=${TOOLCHAIN:-$(date +%Y-%m-%d)} - -function check_git_fixed_subtree() { - if [[ ! -e ./git-fixed-subtree.sh ]]; then - echo "Missing git-fixed-subtree.sh. Please run the following commands to download it:" - echo "curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/bjorn3/git/tqc-subtree-portable/contrib/subtree/git-subtree.sh -o git-fixed-subtree.sh" - echo "chmod u+x git-fixed-subtree.sh" - exit 1 - fi - if [[ ! -x ./git-fixed-subtree.sh ]]; then - echo "git-fixed-subtree.sh is not executable. Please run the following command to make it executable:" - echo "chmod u+x git-fixed-subtree.sh" - exit 1 - fi -} +export GIT_CONFIG_GLOBAL=$(pwd)/../rust/josh.gitconfig +export RUSTC_GIT=../rust case $1 in "prepare") - echo "=> Installing new nightly" - rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists - sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain.toml - echo "=> Uninstalling all old nightlies" - for nightly in $(rustup toolchain list | grep nightly | grep -v "$TOOLCHAIN" | grep -v nightly-x86_64); do + for nightly in $(RUSTUP_AUTO_INSTALL=0 rustup toolchain list | grep nightly | grep -v nightly-x86_64); do rustup toolchain uninstall "$nightly" done + echo "=> Installing new nightly" + rustup toolchain install + ./clean_all.sh ./y.sh prepare ;; - "commit") - git add rust-toolchain.toml - git commit -m "Rustup to $(rustc -V)" - ;; "push") - check_git_fixed_subtree - - cg_clif=$(pwd) - pushd ../rust - git pull origin main + username=${2:-bjorn3} branch=sync_cg_clif-$(date +%Y-%m-%d) - git checkout -b "$branch" - "$cg_clif/git-fixed-subtree.sh" pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/rust-lang/rustc_codegen_cranelift.git main - git push -u my "$branch" - - # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing - # from rust-lang/rust later - "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust - popd - git merge sync_from_rust + rustc-josh-sync push "$branch" "$username" ;; "pull") - check_git_fixed_subtree - - RUST_VERS=$(curl "https://static.rust-lang.org/dist/$TOOLCHAIN/channel-rust-nightly-git-commit-hash.txt") - echo "Pulling $RUST_VERS ($TOOLCHAIN)" - - cg_clif=$(pwd) - pushd ../rust - git fetch origin main - git -c advice.detachedHead=false checkout "$RUST_VERS" - "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust - popd - git merge sync_from_rust -m "Sync from rust $RUST_VERS" - git branch -d sync_from_rust + git checkout -b sync_from_rust + rustc-josh-sync pull ;; *) echo "Unknown command '$1'" - echo "Usage: ./rustup.sh prepare|commit" + echo "Usage: ./rustup.sh prepare|pull|push [fork]" ;; esac diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index 5a1374fe56e09..648f68588e04b 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -164,6 +164,7 @@ rm -r tests/run-make/const-trait-stable-toolchain # same rm -r tests/run-make/print-request-help-stable-unstable # same rm -r tests/run-make/issue-149402-suggest-unresolve # same rm -r tests/run-make/const-destruct-stable-toolchain # same +rm -r tests/run-make/unstable-target-feature # same rm -r tests/run-make/incr-add-rust-src-component rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path rm -r tests/run-make/export # something about rustc version mismatches diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs index ca9157daae584..8e339167a5b12 100644 --- a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs +++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs @@ -135,8 +135,8 @@ builtin_functions! { // float intrinsics fn __powisf2(a: f32, b: i32) -> f32; fn __powidf2(a: f64, b: i32) -> f64; - // FIXME(f16_f128): `compiler-builtins` doesn't currently support `__powitf2` on MSVC. - // fn __powitf2(a: f128, b: i32) -> f128; + #[cfg(not(all(target_os = "windows", target_env = "gnu")))] + fn __powitf2(a: f128, b: i32) -> f128; fn powf(a: f32, b: f32) -> f32; fn pow(a: f64, b: f64) -> f64; fn expf(f: f32) -> f32; diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index 136e98964bc35..f3763b5895eba 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -855,42 +855,59 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( ); } - "llvm.x86.pclmulqdq" => { + "llvm.x86.pclmulqdq" | "llvm.x86.pclmulqdq.256" | "llvm.x86.pclmulqdq.512" => { // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clmulepi64_si128&ig_expand=772 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_clmulepi64_epi128&ig_expand=752 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_clmulepi64_epi128&ig_expand=753 intrinsic_args!(fx, args => (a, b, _imm8); intrinsic); - let a = a.load_scalar(fx); - let b = b.load_scalar(fx); + let (lane_count, intrinsic_name) = match intrinsic { + "llvm.x86.pclmulqdq" => (1, "_mm_clmulepi64_si128"), + "llvm.x86.pclmulqdq.256" => (2, "_mm256_clmulepi64_epi128"), + "llvm.x86.pclmulqdq.512" => (4, "_mm512_clmulepi64_epi128"), + _ => unreachable!(), + }; - let imm8 = - if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[2].node) { - imm8 - } else { - fx.tcx.dcx().span_fatal( - span, - "Index argument for `_mm_clmulepi64_si128` is not a constant", - ); - }; + let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[2].node) else { + fx.tcx.dcx().span_fatal( + span, + format!("Index argument for `{intrinsic_name}` is not a constant"), + ) + }; let imm8 = imm8.to_u8(); - codegen_inline_asm_inner( - fx, - &[InlineAsmTemplatePiece::String(format!("pclmulqdq xmm0, xmm1, {imm8}").into())], - &[ - CInlineAsmOperand::InOut { - reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), - _late: true, - in_value: a, - out_place: Some(ret), - }, - CInlineAsmOperand::In { - reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), - value: b, - }, - ], - InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, - ); + // Cranelift does not currently support 256-bit or 512-bit SSA values, so we use the + // 128-bit instruction on each 128-bit lane. + for lane in 0..lane_count { + let a = a.value_typed_lane(fx, fx.tcx.types.u128, lane).load_scalar(fx); + let b = b.value_typed_lane(fx, fx.tcx.types.u128, lane).load_scalar(fx); + let ret_lane = ret.place_typed_lane(fx, fx.tcx.types.u128, lane); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + format!("pclmulqdq xmm0, xmm1, {imm8}").into(), + )], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86( + X86InlineAsmReg::xmm0, + )), + _late: true, + in_value: a, + out_place: Some(ret_lane), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86( + X86InlineAsmReg::xmm1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } } "llvm.x86.aesni.aeskeygenassist" => {