目前在64GB RAM 的centos 服务器上同时开100个也没有压力
public sealed class PuppeteerSharpUtil
{
private static Browser _browser { get; set; }
/// <summary>
/// 异步线程锁(让浏览器只初始化一次)
/// </summary>
private static SemaphoreSlim slimLock = new SemaphoreSlim(1, 1);
/// <summary>
/// 初始化浏览
/// </summary>
/// <returns></returns>
private async static Task InItBrowser()
{
try
{
await slimLock.WaitAsync();
if (_browser == null)
{
//不存在则下载浏览器
var result = await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
_browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
Args = new string[] {
"--disable-infobars",//隐藏 自动化标题
"--no-sandbox", // 沙盒模式
},
});
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
slimLock.Release();
}
}
/// <summary>
/// 异步线程锁(锁定每次只处理10个任务)
/// </summary>
private static readonly SemaphoreSlim _mutex = new SemaphoreSlim(ConfigExtensions.Configuration["pageCnt"].ToInt(10));
/// <summary>
/// 异步截图
/// </summary>
/// <typeparam name="T1"></typeparam>
/// <param name="url">加载的url地址</param>
/// <param name="savePath">保存路径</param>
/// <param name="isTagName">标识(用于判断网页是否加载某个模块)</param>
/// <param name="action">截图完成后要执行的方法</param>
/// <param name="par1">参数1</param>
/// <param name="width">宽</param>
/// <param name="height">高</param>
public static async Task ScreenshotAsync<T1>(string url, string savePath, string isTagName, Action<T1> action, T1 par1, int width = 1000, int height = 1000)
{
try
{
await _mutex.WaitAsync();
if (_browser == null) await InItBrowser();
using Page page = await _browser.NewPageAsync();
try
{
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
WaitUntilNavigation[] wn = new WaitUntilNavigation[50];
wn[0] = WaitUntilNavigation.DOMContentLoaded;
wn[1] = WaitUntilNavigation.Load;
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
await page.GoToAsync(url, new NavigationOptions() { WaitUntil = wn });
var selector = await page.WaitForSelectorAsync(isTagName, new WaitForSelectorOptions() { Timeout = ConfigExtensions.Configuration["ScreensTimeout"].ToInt(30000) });
if (selector != null)
{
await page.ScreenshotAsync(savePath,
new ScreenshotOptions
{
Type = ScreenshotType.Png,
});
action(par1);
}
else
{
NLogHelp.InfoLog(url + "-----Selector查询不到头部" + selector.ToString() + "-------" + isTagName);
}
}
catch (Exception ex)
{
page.Dispose();
NLogHelp.ErrorLog(url + "-----截图出错", ex);
}
}
catch (Exception ex)
{
NLogHelp.ErrorLog("截图----出错:", ex);
GC.Collect();
throw ex;
}
finally
{
_mutex.Release();
GC.Collect();
}
}
/// <summary>
/// 异步截图
/// </summary>
/// <typeparam name="T1"></typeparam>
/// <param name="url">加载的url地址</param>
/// <param name="savePath">保存路径</param>
/// <param name="isTagName">标识(用于判断网页是否加载某个模块)</param>
/// <param name="action">截图完成后要执行的方法</param>
/// <param name="par1">参数1</param>
/// <param name="par2">参数2</param>
/// <param name="width">宽</param>
/// <param name="height">高</param>
public static async Task ScreenshotAsync<T1, T2>(string url, string savePath, string isTagName, Action<T1, T2> action, T1 par1, T2 par2, int width = 1000, int height = 1000)
{
try
{
await _mutex.WaitAsync();
if (_browser == null) await InItBrowser();
using Page page = await _browser.NewPageAsync();
try
{
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
WaitUntilNavigation[] wn = new WaitUntilNavigation[50];
wn[0] = WaitUntilNavigation.DOMContentLoaded;
wn[1] = WaitUntilNavigation.Load;
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
await page.GoToAsync(url, new NavigationOptions() { WaitUntil = wn });
var selector = await page.WaitForSelectorAsync(isTagName, new WaitForSelectorOptions() { Timeout = ConfigExtensions.Configuration["ScreensTimeout"].ToInt(30000) });
if (selector != null)
{
await page.ScreenshotAsync(savePath,
new ScreenshotOptions
{
Type = ScreenshotType.Png,
});
action?.Invoke(par1, par2);
}
else
{
NLogHelp.InfoLog(url + "-----Selector查询不到头部" + selector.ToString() + "-------" + isTagName);
}
}
catch (Exception ex)
{
page.Dispose();
NLogHelp.ErrorLog(url + "-----截图出错", ex);
}
}
catch (Exception ex)
{
NLogHelp.ErrorLog("截图----出错:", ex);
GC.Collect();
throw ex;
}
finally
{
_mutex.Release();
GC.Collect();
}
}
/// <summary>
/// 异步截图
/// </summary>
/// <typeparam name="T1"></typeparam>
/// <param name="url">加载的url地址</param>
/// <param name="savePath">保存路径</param>
/// <param name="isTagName">标识(用于判断网页是否加载某个模块)</param>
/// <param name="action">截图完成后要执行的方法</param>
/// <param name="par1">参数1</param>
/// <param name="par2">参数2</param>
/// <param name="par3">参数3</param>
/// <param name="width">宽</param>
/// <param name="height">高</param>
public static async Task ScreenshotAsync<T1, T2, T3>(string url, string savePath, string isTagName, Action<T1, T2, T3> action, T1 par1, T2 par2, T3 par3, int width = 1080, int height = 1920)
{
await _mutex.WaitAsync();
if (_browser == null) await InItBrowser();
try
{
await Task.Run(async () =>
{
using Page page = await _browser.NewPageAsync();
try
{
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
WaitUntilNavigation[] wn = new WaitUntilNavigation[50];
wn[0] = WaitUntilNavigation.DOMContentLoaded;
wn[1] = WaitUntilNavigation.Load;
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
await page.GoToAsync(url, new NavigationOptions() { WaitUntil = wn });
var selector = await page.WaitForSelectorAsync(isTagName, new WaitForSelectorOptions() { Timeout = ConfigExtensions.Configuration["ScreensTimeout"].ToInt(30000) });
if (selector != null)
{
await page.ScreenshotAsync(savePath,
new ScreenshotOptions
{
Type = ScreenshotType.Png,
});
action?.Invoke(par1, par2, par3);
}
else
{
NLogHelp.InfoLog(url + "-----Selector查询不到头部" + selector.ToString() + "-------" + isTagName);
}
}
catch (Exception ex)
{
page.Dispose();
NLogHelp.ErrorLog(url + "-----截图出错", ex);
}
});
}
catch (Exception ex)
{
GC.Collect();
NLogHelp.ErrorLog("截图----出错:", ex);
throw ex;
}
finally
{
_mutex.Release();
GC.Collect();
}
}
/// <summary>
/// 异步截图
/// </summary>
/// <typeparam name="T1"></typeparam>
/// <param name="url">加载的url地址</param>
/// <param name="savePath">保存路径</param>
/// <param name="isTagName">标识(用于判断网页是否加载某个模块)</param>
/// <param name="action">截图完成后要执行的方法</param>
/// <param name="par1">参数1</param>
/// <param name="par2">参数2</param>
/// <param name="par3">参数3</param>
/// <param name="par4">参数4</param>
/// <param name="width">宽</param>
/// <param name="height">高</param>
public static async Task ScreenshotAsync<T1, T2, T3, T4>(string url, string savePath, string isTagName, Action<T1, T2, T3, T4> action, T1 par1, T2 par2, T3 par3, T4 par4, int width = 1000, int height = 1000)
{
NLogHelp.InfoLog(url + "截图进来时间----:" + par4 + " -----------" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
await _mutex.WaitAsync();
if (_browser == null) await InItBrowser();
try
{
await Task.Run(async () =>
{
using Page page = await _browser.NewPageAsync();
try
{
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
WaitUntilNavigation[] wn = new WaitUntilNavigation[50];
wn[0] = WaitUntilNavigation.DOMContentLoaded;
wn[1] = WaitUntilNavigation.Load;
await page.SetViewportAsync(new ViewPortOptions
{
Width = width,
Height = height
});
await page.GoToAsync(url, new NavigationOptions() { WaitUntil = wn });
var selector = await page.WaitForSelectorAsync(isTagName, new WaitForSelectorOptions() { Timeout = ConfigExtensions.Configuration["ScreensTimeout"].ToInt(30000) });
if (selector != null)
{
await page.ScreenshotAsync(savePath,
new ScreenshotOptions
{
Type = ScreenshotType.Png,
});
action?.Invoke(par1, par2, par3, par4);
}
else
{
NLogHelp.InfoLog(url + "-----Selector查询不到头部" + selector.ToString() + "-------" + isTagName);
}
}
catch (Exception ex)
{
page.Dispose();
NLogHelp.ErrorLog(url + "-----截图出错", ex);
}
});
}
catch (Exception ex)
{
GC.Collect();
NLogHelp.ErrorLog("截图----出错:", ex);
throw ex;
}
finally
{
_mutex.Release();
GC.Collect();
}
}
}
评论区