guancheng.huang/webman_ad
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始化创建定时任务

Browse Source
This commit is contained in:
hgc 2024-12-17 21:11:21 +08:00
parent bd0b06ee30
commit 1eec1d05f4
17 changed files with 1654 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -4,5 +4,3 @@
/vendor
*.log
.env
/tests/tmp
/tests/.phpunit.result.cache

225
app/event/TiktokAds.php Normal file
View File

@ -0,0 +1,225 @@
<?php
namespace app\event;
use app\model\TiktokAd;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use support\Db;
use QL\QueryList;
use support\Redis;
class TiktokAds
{
//微博热榜地址
// const url = 'https://library.tiktok.com/api/v1/search?region=GB&type=1&start_time=1666540800&end_time=1666627200';
const userAgent = 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
const type = 'tiktokads';
const limit = 12;
const sort_order = 'impression,desc';
// const countries = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IS", "IE","IT", "LV", "LI", "LT", "LU", "MT", "NL", "NO", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "CH", "TR", "GB"] ;
const countries = ["GB", "BE"];
/**
* 每天爬取tiktok广告
* @return void
*/
public function update()
{
try {
$client = new Client([
//允许重定向
// 'allow_redirects' => true,
]);
// 获取前两天 0 点的时间戳
$dayBeforeYesterdayStart = strtotime('-2 days 00:00:00');
// 获取前一天 0 点的时间戳
$yesterdayStart = strtotime('-1 day 00:00:00');
$countryCache = Redis::get(self::type . 'lastCountry');
//全部跑完跳出
if ($countryCache == 'All') {
dump($countryCache . '国家更新tiktok Ads 完成');
return;
}
if (empty($countryCache)) {
$countryCache = 'GB';
}
$searchIdCache = Redis::get(self::type . 'lastSearchId');
$offsetCache = Redis::get(self::type . 'nextOffset');
$totalCache = Redis::get(self::type . 'totalCache');
if (!isset($searchIdCache) || empty($searchIdCache)) {
$searchIdCache = '';
}
if (!isset($offsetCache) || empty($searchIdCache)) {
$offsetCache = 0;
}
if (!isset($totalCache) || empty($searchIdCache)) {
$totalCache = 0;
}
//判断国家是否跑完。
if ($totalCache > 0 && $offsetCache > ceil($totalCache / self::limit)) {
$key = array_search($countryCache, self::countries, true);
if (in_array($countryCache, self::countries) && isset(self::countries[$key + 1])) {
$countryCache = self::countries[$key + 1];
dump('更新' . $countryCache . '国家的tiktok Ads 完成');
} else {
$countryCache = 'All'; //赋值非正常国家的code中断定时任务
dump($countryCache . '国家更新tiktok Ads 完成');
}
return;
}
$start_time = $dayBeforeYesterdayStart;
$end_time = $yesterdayStart;
//当前国家爬取
$currentParams = null;
$currentParams = ['country' => $countryCache, 'search_id' => $searchIdCache, 'type' => 1, 'start_time' => $start_time, 'end_time' => $end_time];
$url = 'https://library.tiktok.com/api/v1/search?region=' . $currentParams['country'] . '&type=' . $currentParams['type'] . '&start_time=' . $currentParams['start_time'] . '&end_time=' . $currentParams['end_time'];
$res = json_decode($client->post($url, [
'headers' => [
'accept' => 'application/json, text/plain, */*',
'accept-language' => 'zh-CN,zh;q=0.9',
'content-type' => 'application/json',
'cookie' => 'cookie: _ttp=2ov8Fc4C2CaNscHJd90O9fMhlpE; _ga=GA1.1.1025820618.1731926196; FPID=FPID2.2.Bcgkp%2Fk%2Bbn5w5YeSMR9wd9VpNHJwTUpkkaEqSdCEa0w%3D.1731926196; FPAU=1.2.944915349.1731926193; FPLC=mbVyryI5aG6IVpAvhs1JsgWjA7FVA6QsCJ7VbXhM7zWoXNp4rcD0IK7FNTTf%2FuOrqeOgqEhTd4NB3hY7q3aDVTGQa3WGHqxkGte4%2BBZxsrpaHFas9kb7DPRXM12T5Q%3D%3D; _ga_TEQXTT9FE4=GS1.1.1732097542.7.0.1732097542.0.0.857840528',
'origin' => 'https://library.tiktok.com',
'priority' => 'u=1, i',
'referer' => 'https://library.tiktok.com/ads?region=AT&start_time=1731945600000&end_time=1732032000000&adv_name=&adv_biz_ids=&query_type=&sort_type=last_shown_date,desc',
'sec-ch-ua' => '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-fetch-dest' => 'empty',
'sec-fetch-mode' => 'cors',
'sec-fetch-site' => 'same-origin',
'user-agent' => self::userAgent,
// 'user-agent' => 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
],
'json' => [
'query' => '',
'query_type' => '',
'adv_biz_ids' => '',
'order' => self::sort_order,
'offset' => (int) $offsetCache,
'search_id' => $currentParams['search_id'],
'limit' => self::limit,
],
])->getBody()->getContents(), true);
// dump($res);return; //调试点
if ($res['search_id'] != $searchIdCache) {
$searchIdCache = $res['search_id'];
dump( 'search_id更新 ' . $searchIdCache . ' 成功');
}
if ($res['total'] == 0 || $res['code'] != 0) {
dump('更新tiktok Ads接口响应异常：' . json_encode($res, JSON_UNESCAPED_UNICODE));
return;
}
$listAdsIds = [];
foreach ($res['data'] as $ad) {
if($ad['audit_status'] == 2 || empty($ad['videos'])){
continue; //审核不过或者没视频不采集
}
$imagesJson = is_array($ad['image_urls']) ? json_encode($ad['image_urls']):json_encode([]);
$rejection_info = is_array($ad['rejection_info']) ? json_encode($ad['rejection_info']):null;
// dump($rejection_info);
$insertData[$ad['id']] = [
'ad_id' => $ad['id'],
'name' => $ad['name'],
'audit_status' => $ad['audit_status'],
'type' => $ad['type'],
'first_shown_date' => $ad['first_shown_date'],
'last_shown_date' => $ad['last_shown_date'],
'image_urls' => $imagesJson,
'estimated_audience' => $ad['estimated_audience'],
'spent' => $ad['spent'],
'impression' => $ad['impression'],
'show_mode' => $ad['show_mode'],
'rejection_info' => $rejection_info,
'sor_audit_status' => $ad['sor_audit_status'],
'country_code' => $countryCache,
];
if(isset($ad['videos']) && !empty($ad['videos'])) {
// 遍历 "videos" 数组
foreach ($ad['videos'] as $video) {
$insertData[$ad['id']]['video_url'] = $video['video_url'];
$insertData[$ad['id']]['cover_img'] = $video['cover_img'];
}
}
$listAdsIds = array_column($insertData, 'ad_id');
}
// dump($insertData);return;
if (empty($insertData)) return;
//开启事务
Db::beginTransaction();
//删除原来的旧数据
TiktokAd::query()->whereIn('ad_id', array_keys($insertData))->delete();
//添加新的数据
TiktokAd::query()->insert($insertData);
//redis缓存
Redis::set(self::type, json_encode($insertData, JSON_UNESCAPED_UNICODE));
//redis缓存 记录更新时间
$time = date('Y-m-d H:i:s');
Redis::set(self::type . 'time', $time);
Redis::set(self::type . 'lastCountry', $countryCache);
Redis::set(self::type . 'nextOffset', ++$offsetCache); //记录下一次的offset
Redis::set(self::type . 'totalCache', $res['total']);
Redis::set(self::type . 'lastSearchId', $searchIdCache);
if(!empty($listAdsIds)){
Redis::rPush(self::type . 'AdsIds',...$listAdsIds);
}
//提交事务
Db::commit();
//销毁$res
unset($res);
dump(date('Y-m-d H:i:s') . '更新' . self::type . '成功');
} catch (GuzzleException|\Exception $exception) {
//回滚事务
Db::rollBack();
dump('更' . self::type . '异常：' . $exception->getMessage());
dump($exception);
}
// } catch (ClientExceptionInterface $e) {
// // 捕获 4xx 错误
// dump( 'Client error: ' . $e->getMessage() . "\n");
// } catch (ServerExceptionInterface $e) {
// // 捕获 5xx 错误
// dump('Server error: ' . $e->getMessage() . "\n");
// } catch (TransportExceptionInterface $e) {
// // 捕获网络传输错误
// dump('Transport error: ' . $e->getMessage() . "\n") ;
// } catch (\Exception $e) {
// // 捕获所有其他错误
// dump('General error: ' . $e->getMessage() . "\n") ;
// }
}
}

184
app/event/TiktokAdsDetails.php Normal file
View File

@ -0,0 +1,184 @@
<?php
namespace app\event;
use app\model\TiktokAdsDetail;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use support\Db;
use QL\QueryList;
use support\Redis;
class TiktokAdsDetails
{
const userAgent = 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
const url = 'https://library.tiktok.com/api/v1/items';
const type = 'tiktokadsdetails';
// const countries = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IS", "IE","IT", "LV", "LI", "LT", "LU", "MT", "NL", "NO", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "CH", "TR", "GB"] ;
// const countries = ["GB", "BE"];
/**
* 每天爬取tiktok广告详情
* @return void
*/
public function update()
{
try {
$client = new Client([
//允许重定向
// 'allow_redirects' => true,
]);
$adIdCache = Redis::lPop(TiktokAds::type . 'AdsIds');
if (empty($adIdCache)) {
dump('获取AD id 异常：');
return;
}
$currentParams = ['ad_id' => $adIdCache,'action' => 'details'];
$environment = 'vvv';
$proxyIp = '103.122.176.175';
$proxyPort = '31186';
$proxyUser = 'B978321E';
$proxyPassword = '1EC4E3C7398F';
$proxyAuth = base64_encode($proxyUser . ":" . $proxyPassword);
$options = [
"headers" => [
'accept' => 'application/json, text/plain, */*',
'accept-language' => 'zh-CN,zh;q=0.9',
'content-type' => 'application/json',
'cookie' => 'cookie: _ttp=2ov8Fc4C2CaNscHJd90O9fMhlpE; _ga=GA1.1.1025820618.1731926196; FPID=FPID2.2.Bcgkp%2Fk%2Bbn5w5YeSMR9wd9VpNHJwTUpkkaEqSdCEa0w%3D.1731926196; FPAU=1.2.944915349.1731926193; FPLC=CfAlWGYPInwi1oHk0kTm68d0p21YS3UZ31sOR60H2uVC7A20NL46YHkF9z36OOlKC9XHODO2%2Biet2kk486i6SmO0TcqGntq1CbgwSqOH6f4ZES7jiHeI0mu82CKUVg%3D%3D',
'origin' => 'https://library.tiktok.com',
'priority' => 'u=1, i',
'referer' => 'https://library.tiktok.com/ads?region=AT&start_time=1731945600000&end_time=1732032000000&adv_name=&adv_biz_ids=&query_type=&sort_type=last_shown_date,desc',
'sec-ch-ua' => '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-fetch-dest' => 'empty',
'sec-fetch-mode' => 'cors',
'sec-fetch-site' => 'same-origin',
'user-agent' => self::userAgent,
],
];
if($environment !== 'env') {
$options["headers"]["Proxy-Authorization"] = "Basic " . $proxyAuth;
$options["proxy"] = $proxyIp . ':' . $proxyPort;
}
$res = json_decode($client->request('Get', self::url.'/'.$currentParams['ad_id'].'/'.$currentParams['action'], $options)->getBody()->getContents(), true);
// dump($res);return; //调试点
if (empty($res['data']) || $res['code'] != 0) {
dump('更新tiktok Ads Details接口响应异常：' . json_encode($res, JSON_UNESCAPED_UNICODE));
return;
}
$ad = $res['data'];
if( $ad['ad']['audit_status'] == 2 || empty( $ad['ad']['videos'])){
return; //审核不过或者没视频不采集
}
$imagesJson = is_array($ad['ad']['image_urls']) ? json_encode($ad['ad']['image_urls']):json_encode([]);
$targetingLocationJson = is_array($ad['targeting']['location']) ? json_encode($ad['targeting']['location']):json_encode([]);
$targetingAgeJson = is_array($ad['targeting']['age']) ? json_encode($ad['targeting']['age']):json_encode([]);
$targetingGenderJson = is_array($ad['targeting']['gender']) ? json_encode($ad['targeting']['gender']):json_encode([]);
// dump($rejection_info);
$insertData[$ad['ad']['id']] = [
'ad_id' => $ad['ad']['id'],
'ad_name' => $ad['ad']['name'],
'audit_status' => $ad['ad']['audit_status'],
'ad_type' => $ad['ad']['type'],
'first_shown_date' => $ad['ad']['first_shown_date'],
'last_shown_date' => $ad['ad']['last_shown_date'],
'estimated_audience' => $ad['ad']['estimated_audience'],
'spent' => $ad['ad']['spent'],
'impression' => $ad['ad']['impression'],
'show_mode' => $ad['ad']['show_mode'],
'sor_audit_status' => $ad['ad']['sor_audit_status'],
'image_urls' => $imagesJson,
'advertiser_name' => $ad['advertiser']['name'],
'adv_biz_ids' => $ad['advertiser']['adv_biz_ids'],
'registry_location' => $ad['advertiser']['registry_location'],
'sponsor' => $ad['advertiser']['sponsor'],
'targeting_location' => $targetingLocationJson,
'targeting_age' => $targetingAgeJson,
'targeting_gender' => $targetingGenderJson,
'target_audience_size' => $ad['targeting']['target_audience_size'],
'audience_type' => $ad['targeting']['audience'],
'interest' => $ad['targeting']['interest'],
'video_interactions' => $ad['targeting']['video_interactions'],
'creator_interactions' => $ad['targeting']['creator_interactions'],
];
if(isset($ad['ad']['videos']) && !empty($ad['ad']['videos'])) {
// 遍历 "videos" 数组
foreach ($ad['ad']['videos'] as $video) {
$insertData[$ad['ad']['id']]['video_url'] = $video['video_url'];
$insertData[$ad['ad']['id']]['cover_img'] = $video['cover_img'];
}
}
// dump($insertData);return;
if (empty($insertData)) return;
//开启事务
Db::beginTransaction();
//删除原来的旧数据
TiktokAdsDetail::query()->where('ad_id', $currentParams['ad_id'])->delete();
//添加新的数据
TiktokAdsDetail::query()->insert($insertData);
//redis缓存
Redis::set(self::type, json_encode($insertData, JSON_UNESCAPED_UNICODE));
//redis缓存 记录更新时间
$time = date('Y-m-d H:i:s');
Redis::set(self::type . 'time', $time);
//提交事务
Db::commit();
//销毁$res
unset($res);
dump(date('Y-m-d H:i:s') . '更新' . self::type . '成功');
} catch (GuzzleException|\Exception $exception) {
//回滚事务
Db::rollBack();
dump('更' . self::type . '的广告ID为 '.$currentParams['ad_id'].' 异常：' . $exception->getMessage());
dump($exception);
}
// } catch (ClientExceptionInterface $e) {
// // 捕获 4xx 错误
// dump( 'Client error: ' . $e->getMessage() . "\n");
// } catch (ServerExceptionInterface $e) {
// // 捕获 5xx 错误
// dump('Server error: ' . $e->getMessage() . "\n");
// } catch (TransportExceptionInterface $e) {
// // 捕获网络传输错误
// dump('Transport error: ' . $e->getMessage() . "\n") ;
// } catch (\Exception $e) {
// // 捕获所有其他错误
// dump('General error: ' . $e->getMessage() . "\n") ;
// }
}
}

54
app/process/UpdateGoogleAdsTask.php Normal file
View File

@ -0,0 +1,54 @@
<?php
namespace app\process;
use app\event\TiktokAds;
use app\event\TiktokAdsDetails;
use Webman\Event\Event;
use Workerman\Crontab\Crontab;
/**
* 更新热搜列表任务
*/
class UpdateGoogleAdsTask
{
//错开时间执行，否则固定时间段接口会响应很慢
public function onWorkerStart()
{
// 每15分钟执行一次
new Crontab('*/15 * * * * *', function () {
// dump(date('Y-m-d H:i:s') . '更新' . TiktokAdsDetails::type . '开始');
// Event::emit(TiktokAdsDetails::type, null);
}
);
// 每12分钟执行一次
new Crontab('0 */12 * * * *', function () {
// dump(date('Y-m-d H:i:s') . '更新' . HuPu::type . '开始');
// Event::emit(HuPu::type, null);
// dump(date('Y-m-d H:i:s') . '更新' . DouBan::type . '开始');
// Event::emit(DouBan::type, null);
//
// dump(date('Y-m-d H:i:s') . '更新' . Itzhijia::type . '开始');
// Event::emit(Itzhijia::type, null);
});
// 每30分钟执行一次
// new Crontab('0 */30 * * * *', function () {
// dump(date('Y-m-d H:i:s') . '更新' . V2ex::type . '开始');
// Event::emit(V2ex::type, null);
//
// dump(date('Y-m-d H:i:s') . '更新' . GitHub::type . '开始');
// Event::emit(GitHub::type, null);
//
// dump(date('Y-m-d H:i:s') . '更新' . JueJin::type . '开始');
// Event::emit(JueJin::type, null);
// });
}
}

6
composer.json
View File

@ -35,7 +35,11 @@
"webman/auto-route": "^1.0",
"psr/container": "^1.1.1",
"php-di/php-di": "^6.3",
"doctrine/annotations": "^1.14"
"doctrine/annotations": "^1.14",
"webman/redis-queue": "^1.3",
"webman/event": "^1.0",
"workerman/crontab": "^1.0",
"illuminate/redis": "^10.48"
},
"suggest": {
"ext-event": "For better performance. "

1068
composer.lock generated
View File

File diff suppressed because it is too large Load Diff

17
config/event.php Normal file
View File

@ -0,0 +1,17 @@
<?php
use app\event\TiktokAds;
use app\event\TiktokAdsDetails;
return [
//知乎热榜
TiktokAds::type => [
[TiktokAds::class, 'update'],
],
TiktokAdsDetails::type => [
[TiktokAdsDetails::class, 'update'],
],
];

4
config/plugin/webman/event/app.php Normal file
View File

@ -0,0 +1,4 @@
<?php
return [
'enable' => true,
];

17
config/plugin/webman/event/bootstrap.php Normal file
View File

@ -0,0 +1,17 @@
<?php
/**
* This file is part of webman.
*
* Licensed under The MIT License
* For full copyright and license information, please see the MIT-LICENSE.txt
* Redistributions of files must retain the above copyright notice.
*
* @author walkor<walkor@workerman.net>
* @copyright walkor<walkor@workerman.net>
* @link http://www.workerman.net/
* @license http://www.opensource.org/licenses/mit-license.php MIT License
*/
return [
Webman\Event\BootStrap::class,
];

7
config/plugin/webman/event/command.php Normal file
View File

@ -0,0 +1,7 @@
<?php
use Webman\Event\EventListCommand;
return [
EventListCommand::class
];

4
config/plugin/webman/redis-queue/app.php Normal file
View File

@ -0,0 +1,4 @@
<?php
return [
'enable' => true,
];

7
config/plugin/webman/redis-queue/command.php Normal file
View File

@ -0,0 +1,7 @@
<?php
use Webman\RedisQueue\Command\MakeConsumerCommand;
return [
MakeConsumerCommand::class
];

32
config/plugin/webman/redis-queue/log.php Normal file
View File

@ -0,0 +1,32 @@
<?php
/**
* This file is part of webman.
*
* Licensed under The MIT License
* For full copyright and license information, please see the MIT-LICENSE.txt
* Redistributions of files must retain the above copyright notice.
*
* @author walkor<walkor@workerman.net>
* @copyright walkor<walkor@workerman.net>
* @link http://www.workerman.net/
* @license http://www.opensource.org/licenses/mit-license.php MIT License
*/
return [
'default' => [
'handlers' => [
[
'class' => Monolog\Handler\RotatingFileHandler::class,
'constructor' => [
runtime_path() . '/logs/redis-queue/queue.log',
7, //$maxFiles
Monolog\Logger::DEBUG,
],
'formatter' => [
'class' => Monolog\Formatter\LineFormatter::class,
'constructor' => [null, 'Y-m-d H:i:s', true],
],
]
],
]
];

11
config/plugin/webman/redis-queue/process.php Normal file
View File

@ -0,0 +1,11 @@
<?php
return [
'consumer' => [
'handler' => Webman\RedisQueue\Process\Consumer::class,
'count' => 8, // 可以设置多进程同时消费
'constructor' => [
// 消费者类目录
'consumer_dir' => app_path() . '/queue/redis'
]
]
];

13
config/plugin/webman/redis-queue/redis.php Normal file
View File

@ -0,0 +1,13 @@
<?php
return [
'default' => [
'host' => 'redis://127.0.0.1:6379',
'options' => [
'auth' => null, // 密码，字符串类型，可选参数
'db' => 0, // 数据库
'prefix' => '', // key 前缀
'max_attempts' => 5, // 消费失败后，重试次数
'retry_seconds' => 5, // 重试间隔，单位秒
]
],
];

4
config/process.php
View File

@ -58,5 +58,9 @@ return [
'enable_memory_monitor' => DIRECTORY_SEPARATOR === '/',
]
]
],
'updateGoogleAdsList' => [
'handler' => app\process\UpdateGoogleAdsTask::class
]
];

6
config/redis.php
View File

@ -14,9 +14,9 @@
return [
'default' => [
'host' => '127.0.0.1',
'password' => null,
'host' => getenv('REDIS_HOST'),
'password' => getenv('REDIS_PASSWORD'),
'port' => 6379,
'database' => 0,
'database' => 9,
],
];