I'm trying to have my C++ DX11 engine render to a window that was created by an editor written in C#.

I'm using FindWindow to get the HWND:

HRESULT initWindow( HINSTANCE hInstance, int nCmdShow )
{
    g_WindowHandle = 0;
    g_WindowHandle = FindWindow( 0, L"Editor" );

    if( g_WindowHandle )
        return S_OK;

    // more code to create own window if editor isn't running
}

And it works as expected, i can, for example, resize the window from the engine side without problems.

I then pass the window handle to my renderer to create the D3DDevice and swapchain:

HRESULT RendererDX11::initD3DDevice( const HWND windowHandle )
{
    UINT createDeviceFlags = 0;

#ifdef _DEBUG
    createDeviceFlags |= D3D11_CREATE_DEVICE_DEBUG;
#endif

    D3D_DRIVER_TYPE driverTypes[] = { D3D_DRIVER_TYPE_HARDWARE, 
                                      D3D_DRIVER_TYPE_REFERENCE };
    UINT numDriverTypes = ARRAYSIZE( driverTypes );

    D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_0, 
                                          D3D_FEATURE_LEVEL_10_1, 
                                          D3D_FEATURE_LEVEL_10_0 };
    UINT numFeatureLevels = ARRAYSIZE( featureLevels );

    DXGI_SWAP_CHAIN_DESC sd;
    ZeroMemory( &sd, sizeof(sd) );
    sd.BufferCount                        = m_BufferCount;
    sd.BufferDesc.Width                   = m_DisplayResX;
    sd.BufferDesc.Height                  = m_DisplayResY;
    sd.BufferDesc.Format                  = DXGI_FORMAT_R8G8B8A8_UNORM;
    sd.BufferDesc.RefreshRate.Numerator   = 0;
    sd.BufferDesc.RefreshRate.Denominator = 0;
    sd.BufferDesc.Scaling                 = DXGI_MODE_SCALING_UNSPECIFIED;
    sd.BufferDesc.ScanlineOrdering        = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED;
    sd.BufferUsage                        = DXGI_USAGE_RENDER_TARGET_OUTPUT;
    sd.Flags                              = m_Fullscreen ? DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH : 0;
    sd.OutputWindow                       = windowHandle;
    sd.SampleDesc.Count                   = 1;
    sd.SampleDesc.Quality                 = 0;
    sd.Windowed                           = m_Fullscreen ? FALSE : TRUE;
    sd.SwapEffect                         = DXGI_SWAP_EFFECT_DISCARD;
    
    HRESULT hr = S_OK;

    for( UINT driverTypeIndex = 0; driverTypeIndex < numDriverTypes; driverTypeIndex++ )
    {
        m_driverType = driverTypes[driverTypeIndex];
        hr = D3D11CreateDeviceAndSwapChain(          // fails when using the editor's window
                nullptr,
                m_driverType, 
                nullptr, 
                createDeviceFlags, 
                featureLevels, 
                numFeatureLevels,
                D3D11_SDK_VERSION, 
                &sd, 
                &m_pSwapChain, 
                &m_pD3Ddevice, 
                &m_featureLevel, 
                &m_pImmediateContext );

        if( SUCCEEDED(hr) ) break;
    }

    if( FAILED(hr) )
    {
        SHOW_ERROR( hr );      // 0x887A0001: DXGI_ERROR_INVALID_CALL
        return hr;
    }

    return S_OK;
}

And it only works as long as i'm using a window that the engine created itself.

I'm getting the DXGI_ERROR_INVALID_CALL error when using the editor's window. The error description on MSDN just reads:

The application provided invalid parameter data; this must be debugged and fixed before the application is released.

Which doesn't really help.

I'm pretty much out of ideas now, is there anything i'm missing?

I just figured it out, the engine window needed to be created as a child to the editor window :)

You can do that by passing the other application's window handle as hwndParent and the window style WS_CHILD to CreateWindow.