CastReg.h

Go to the documentation of this file.

// CastReg.h:
// ------------
//

#ifndef __IPSDKUTIL_INSTRUCTIONSET_DETAIL_AVX512_CASTREG_H__
#define __IPSDKUTIL_INSTRUCTIONSET_DETAIL_AVX512_CASTREG_H__

#include <IPSDKUtil/IPSDKUtilExports.h>

#include <IPSDKUtil/InstructionSet/detail/CastReg.h>
#include <IPSDKUtil/InstructionSet/Comparison/detail/AVX512/IsLessRegDecl.h>
#include <IPSDKUtil/InstructionSet/Logical/detail/MaskBlendReg.h>
#include <IPSDKUtil/InstructionSet/Logical/detail/AllBitsToZeroReg.h>
#include <IPSDKUtil/InstructionSet/Logical/detail/AllBitsToOneReg.h>
#include <IPSDKUtil/Tools/ForceInline.h>

#include <boost/mpl/and.hpp>
#include <boost/mpl/or.hpp>
#include <boost/mpl/sizeof.hpp>

#include "immintrin.h"

namespace ipsdk {
namespace simd {
namespace detail {


template <typename TIn, typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, TIn, TOut,
    typename boost::enable_if<
        typename boost::mpl::or_<
            typename boost::is_same<TIn, TOut>::type,
            typename boost::mpl::and_<
                typename boost::is_integral<TIn>::type,
                typename boost::mpl::and_<
                    typename boost::is_integral<TOut>::type,
                    typename boost::mpl::equal_to<
                        boost::mpl::sizeof_<TIn>,
                        boost::mpl::sizeof_<TOut>
                    >::type
                >::type
            >::type
        >::type
    >::type
>
{
    static IPSDK_FORCEINLINE
    void act(const typename Avx512Type<TIn>::Type& in, 
             typename Avx512Type<TOut>::Type& out)
    {
        out = in;
    }

    static IPSDK_FORCEINLINE
    void act(const typename Avx512MaskType<TIn>::Type& in,
             typename Avx512Type<TOut>::Type& out)
    {
        out = MaskBlendReg<eInstructionSet::eIS_Avx512, TIn>::act(
            in,
            AllBitsToZeroReg<eInstructionSet::eIS_Avx512, TIn>::act(),
            AllBitsToOneReg<eInstructionSet::eIS_Avx512, TIn>::act());
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipUInt8, TOut,
    typename boost::enable_if<
        typename boost::mpl::equal_to<
            boost::mpl::int_<sizeof(TOut)>, 
            boost::mpl::int_<2>
        >::type
    >::type
>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipUInt8>::Type& in, 
             typename Avx512Type<TOut>::Type& outl,
             typename Avx512Type<TOut>::Type& outh)
    {
        __m512i inPermuted = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 3, 6, 2, 5, 1, 4, 0), in);
        outl = _mm512_unpacklo_epi8(inPermuted, _mm512_set1_epi8(0));
        outh = _mm512_unpackhi_epi8(inPermuted, _mm512_set1_epi8(0));
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipInt8, TOut,
    typename boost::enable_if_c<sizeof(TOut)==2>::type>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipInt8>::Type& in, 
             typename Avx512Type<TOut>::Type& outl,
             typename Avx512Type<TOut>::Type& outh)
    {
        __m512i inPermuted = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 3, 6, 2, 5, 1, 4, 0), in);
        outl = _mm512_unpacklo_epi8(inPermuted, inPermuted);
        outl = _mm512_srai_epi16(outl, 8);
        outh = _mm512_unpackhi_epi8(inPermuted, inPermuted);
        outh = _mm512_srai_epi16(outh, 8);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipInt32, ipReal32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipInt32>::Type& in,
             Avx512Type<ipReal32>::Type& out)
    {
        out = _mm512_cvtepi32_ps(in);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipUInt32, ipReal32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipUInt32>::Type& in,
             Avx512Type<ipReal32>::Type& out)
    {
        out = _mm512_cvtepu32_ps(in);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal32, ipInt32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal32>::Type& in,
             Avx512Type<ipInt32>::Type& out)
    {
        out = _mm512_cvttps_epi32(in);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal32, ipUInt32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal32>::Type& in,
             Avx512Type<ipUInt32>::Type& out)
    {
        out = _mm512_cvttps_epu32(in);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipInt32, ipReal64>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipInt32>::Type& in,
             Avx512Type<ipReal64>::Type& outl,
             Avx512Type<ipReal64>::Type& outh)
    {
        outl = _mm512_cvtepi32_pd(_mm512_castsi512_si256(in));
        outh = _mm512_cvtepi32_pd(_mm512_extracti64x4_epi64(in, 1));
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipUInt32, ipReal64>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipUInt32>::Type& in,
             Avx512Type<ipReal64>::Type& outl,
             Avx512Type<ipReal64>::Type& outh)
    {
        outl = _mm512_cvtepu32_pd(_mm512_castsi512_si256(in));
        outh = _mm512_cvtepu32_pd(_mm512_extracti64x4_epi64(in, 1));
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal64, ipInt32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal64>::Type& inl,
             const Avx512Type<ipReal64>::Type& inh,
             Avx512Type<ipInt32>::Type& out)
    {
        const __m256i outl = _mm512_cvttpd_epi32(inl);
        const __m256i outh = _mm512_cvttpd_epi32(inh);
        out = _mm512_inserti64x4(out, outl, 0);
        out = _mm512_inserti64x4(out, outh, 1);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal64, ipUInt32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal64>::Type& inl,
             const Avx512Type<ipReal64>::Type& inh,
             Avx512Type<ipUInt32>::Type& out)
    {
        const __m256i outl = _mm512_cvttpd_epu32(inl);
        const __m256i outh = _mm512_cvttpd_epu32(inh);
        out = _mm512_inserti64x4(out, outl, 0);
        out = _mm512_inserti64x4(out, outh, 1);
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipInt32, TOut,
    typename boost::enable_if_c<sizeof(TOut)==8 &&
    boost::is_integral<TOut>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipInt32>::Type& in,
             typename Avx512Type<TOut>::Type& outl,
             typename Avx512Type<TOut>::Type& outh)
    {
        __m512i inPermuted = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 3, 6, 2, 5, 1, 4, 0), in);
        
        const Avx512Type<ipInt32>::Type zero = _mm512_setzero_si512();
        Avx512MaskType<ipInt32>::Type hiMask;
        IsLessReg<eInstructionSet::eIS_Avx512, ipInt32>::act(inPermuted, zero, hiMask);
        Avx512Type<ipInt32>::Type hi;
        CastReg<eInstructionSet::eIS_Avx512, ipInt32, ipInt32>::act(hiMask, hi);
        outl = _mm512_unpacklo_epi32(inPermuted, hi);
        outh = _mm512_unpackhi_epi32(inPermuted, hi);
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipUInt32, TOut,
    typename boost::enable_if_c<sizeof(TOut)==8 &&
    boost::is_integral<TOut>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipUInt32>::Type& in,
             typename Avx512Type<TOut>::Type& outl,
             typename Avx512Type<TOut>::Type& outh)
    {
        __m512i inPermuted = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 3, 6, 2, 5, 1, 4, 0), in);
        const Avx512Type<ipUInt32>::Type zero = _mm512_setzero_si512();
        outl = _mm512_unpacklo_epi32(inPermuted, zero);
        outh = _mm512_unpackhi_epi32(inPermuted, zero);
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipInt16, TOut,
    typename boost::enable_if_c<sizeof(TOut)==4
    && boost::is_integral<TOut>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipInt16>::Type& in,
             typename Avx512Type<TOut>::Type& outl,
             typename Avx512Type<TOut>::Type& outh)
    {
        __m512i inPermuted = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 3, 6, 2, 5, 1, 4, 0), in);

        outl = _mm512_unpacklo_epi16(inPermuted, inPermuted);
        outh = _mm512_unpackhi_epi16(inPermuted, inPermuted);
        outl = _mm512_srai_epi32(outl, 16);
        outh = _mm512_srai_epi32(outh, 16);
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipUInt16, TOut,
    typename boost::enable_if_c<sizeof(TOut)==4 &&
    boost::is_integral<TOut>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipUInt16>::Type& in,
             typename Avx512Type<TOut>::Type& outl,
             typename Avx512Type<TOut>::Type& outh)
    {
        __m512i inPermuted = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 3, 6, 2, 5, 1, 4, 0), in);

        outl = _mm512_unpacklo_epi16(inPermuted, _mm512_set1_epi16(0));
        outh = _mm512_unpackhi_epi16(inPermuted, _mm512_set1_epi16(0));
    }
};

template <typename TIn>
struct CastReg<eInstructionSet::eIS_Avx512, TIn, ipReal32,
    typename boost::enable_if_c<sizeof(TIn)==2 && 
    boost::is_integral<TIn>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const typename Avx512Type<TIn>::Type& in,
             Avx512Type<ipReal32>::Type& outl,
             Avx512Type<ipReal32>::Type& outh)
    {
        Avx512Type<ipInt32>::Type in32l, in32h;
        CastReg<eInstructionSet::eIS_Avx512, TIn, ipInt32>::act(in, in32l, in32h);
        CastReg<eInstructionSet::eIS_Avx512, ipInt32, ipReal32>::act(in32l, outl);
        CastReg<eInstructionSet::eIS_Avx512, ipInt32, ipReal32>::act(in32h, outh);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal32, ipReal64>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal32>::Type& in,
             Avx512Type<ipReal64>::Type& outl,
             Avx512Type<ipReal64>::Type& outh)
    {
        outl = _mm512_cvtps_pd(_mm512_castps512_ps256(in));
        outh = _mm512_cvtps_pd(_mm512_castps512_ps256(_mm512_shuffle_f32x4(in, in, 0x4E)));
    }
};

template <typename TIn>
struct CastReg<eInstructionSet::eIS_Avx512, TIn, ipUInt8,
    typename boost::enable_if_c<sizeof(TIn)==2>::type
>
{
    static IPSDK_FORCEINLINE
    void act(const typename Avx512Type<TIn>::Type& inl,
             const typename Avx512Type<TIn>::Type& inh,
             Avx512Type<ipUInt8>::Type& out)
    {
        out = _mm512_packus_epi16(inl, inh);
        out = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 5, 3, 1, 6, 4, 2, 0), out);
    }
};

template <typename TIn>
struct CastReg<eInstructionSet::eIS_Avx512, TIn, ipInt8,
    typename boost::enable_if_c<sizeof(TIn)==2>::type>
{
    static IPSDK_FORCEINLINE
    void act(const typename Avx512Type<TIn>::Type& inl, 
             const typename Avx512Type<TIn>::Type& inh,
             Avx512Type<ipInt8>::Type& out)
    {
        out = _mm512_packs_epi16(inl, inh);
        out = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 5, 3, 1, 6, 4, 2, 0), out);
    }
};

template <typename TIn>
struct CastReg<eInstructionSet::eIS_Avx512, TIn, ipInt16,
    typename boost::enable_if_c<sizeof(TIn)==4 && boost::is_integral<TIn>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const typename Avx512Type<TIn>::Type& inl,
             const typename Avx512Type<TIn>::Type& inh,
             Avx512Type<ipInt16>::Type& out)
    {
        out = _mm512_packs_epi32(inl, inh);
        out = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 5, 3, 1, 6, 4, 2, 0), out);
    }
};

template <typename TIn>
struct CastReg<eInstructionSet::eIS_Avx512, TIn, ipUInt16,
    typename boost::enable_if_c<sizeof(TIn)==4 && boost::is_integral<TIn>::value>::type>
{
    static IPSDK_FORCEINLINE
    void act(const typename Avx512Type<TIn>::Type& inl,
             const typename Avx512Type<TIn>::Type& inh,
             Avx512Type<ipUInt16>::Type& out)
    {
        out = _mm512_packus_epi32(inl, inh);
        out = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 5, 3, 1, 6, 4, 2, 0), out);
    }
};

template <typename TOut>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal32, TOut,
    typename boost::enable_if_c<sizeof(TOut)==2>::type>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal32>::Type& inl,
             const Avx512Type<ipReal32>::Type& inh,
             typename Avx512Type<TOut>::Type& out)
    {
        Avx512Type<ipInt32>::Type inlInt32, inhInt32;
        CastReg<eInstructionSet::eIS_Avx512, ipReal32, ipInt32>::act(
            inl, inlInt32);
        CastReg<eInstructionSet::eIS_Avx512, ipReal32, ipInt32>::act(
            inh, inhInt32);
        CastReg<eInstructionSet::eIS_Avx512, ipInt32, TOut>::act(
            inlInt32, inhInt32, out);
    }
};

template <>
struct CastReg<eInstructionSet::eIS_Avx512, ipReal64, ipReal32>
{
    static IPSDK_FORCEINLINE
    void act(const Avx512Type<ipReal64>::Type& inl,
             const Avx512Type<ipReal64>::Type& inh,
             Avx512Type<ipReal32>::Type& out)
    {
        //out = _mm512_castps256_ps512(_mm512_cvtpd_ps(inl));
        //out = _mm512_insertf256_ps(out, _mm512_cvtpd_ps(inh), 1);
        const Avx512Type<ipReal32>::Type inl_ps = _mm512_castps256_ps512(_mm512_cvtpd_ps(inl));
        const Avx512Type<ipReal32>::Type inh_ps = _mm512_castps256_ps512(_mm512_cvtpd_ps(inh));
        out = _mm512_shuffle_f32x4(inl_ps, inh_ps, 0x44);
    }
};


} // end of namespace detail
} // end of namespace simd
} // end of namespace ipsdk

#endif // __IPSDKUTIL_INSTRUCTIONSET_DETAIL_AVX512_CASTREG_H__