This is something like GlideModal or GlideWindow. It expects a object as
part of its call;

var theObject = {
    id    : 'test-id',//seems to be the ui page it uses
    title : '',//self explanatory
    width : '',//self explanatory
    height: '',//self explanatory
    draggable: true,
    form: 'incident',
    iframe: '',
    onAfterLoad: null,//expects a function, see UI Script "KBViewArticle" for example
    onAfterClose: null,//expecs a function, see ui action "Add" on cmdb_outage_ci_mtom
    closeOnEscape: true,
    showClose: true,
    messages: "",
    allowOverFlowX: true,
};
var go = new GlideOverlay(theObject);
// you can also set things...
go.setBodyFromForm('someuipage');
go.setPreference('sysparm_error','error msg');
go.setAfterLoad(null);//expects function
go.setTitle('the new title');
go.center();
go.render();
/**
// I found these by running this in console
// on my browser
// var a = new GlideOverlay({});
// for(var p in a){
//   if(typeof a[p] === 'function'){
//    console.log('go.' + p + '();');
//   }
// }
**/

// GlideOverlay.close('test-id');
// window.parent.GlideOverlay.close('test-id');

IFrame example

var o = new GlideOverlay({
    title : "Edit Metadata Request",
    iframe : metadataservice_gui_url,
    allowOverflowX : true,
    height : 650,
    width : 1000
    //messages : ""
});

UI Page example

new GlideOverlay({
  title: new GwtMessage().getMessage('Run Fix Script'),
  form: 'run_fix_script_dialog',
  preferences: {
  'sysparm_script_name': gr.u_notes,
  'sysparm_sys_id': gr.sys_id
  },
  width: 600
  }).render();

Further Reading:

• https://community.servicenow.com/community?id=community_question&sys_id=36508924dbbd13800e3dfb651f96196f&view_source=searchResult
• https://community.servicenow.com/community?id=community_question&sys_id=25a30f65dbd8dbc01dcaf3231f961951&view_source=searchResult