During my few last project I had to implement the same pattern:

• PCF control opens the Modal dialog
• Dialog has controls that use OOB lookupObjects function to enter the data

Challenge – the panel that pops up when lookupObjects is executed is located under the initiating Dialog:

In order to fix this issue it would be required to update modalProps property of Dialog:

modalProps={{
	isBlocking: true,
	layerProps: {
		styles: {
			root: {
				zIndex: 0
			}
		}
	}
}}

Once that update is applied, the lookup dialog will appear on top of our custom dialog. Here is an example of the component I used:

import { DefaultButton, Dialog, DialogFooter, DialogType, PrimaryButton } from '@fluentui/react';
import * as React from 'react';

export interface IModalProps {
  utils: ComponentFramework.Utility
}

export const ModalComponent: React.FunctionComponent<IModalProps> = props => {
  const [dialogHidden, setDialogHidden] = React.useState<boolean>(true);

  const dialogContentProps = {
    type: DialogType.normal,
    title: 'Dialog header',
    subText: 'Additional explanation text',
  };

  return (
    <>
      <PrimaryButton text='Open Dialog' onClick={() => { setDialogHidden(false); }} />
      <Dialog
        hidden={dialogHidden}
        onDismiss={() => { setDialogHidden(true); }}
        dialogContentProps={dialogContentProps}
        modalProps={{
          isBlocking: true,
          layerProps: {
            styles: {
              root: {
                zIndex: 0
              }
            }
          }
        }}
      >
        <PrimaryButton
          text='Open Lookup'
          onClick={() => {
            props.utils.lookupObjects({
              allowMultiSelect: false,
              defaultEntityType: "account",
              entityTypes: ["account"],
              viewIds: ["00000000-0000-0000-00aa-000010001001"],
              defaultViewId: "00000000-0000-0000-00aa-000010001001"
            }).then((result: ComponentFramework.LookupValue[]) => {
              console.log(result);
            })
          }}
        />
        <DialogFooter>
          <PrimaryButton onClick={() => { setDialogHidden(true); }} text="Ok" />
          <DefaultButton onClick={() => { setDialogHidden(true); }} text="Cancel" />
        </DialogFooter>
      </Dialog>
    </>
  );
}

Cover photo by Scott Graham on Unsplash