Skip to content

C
crawler.vmusicchart.vn
Commit f63f3efd by Phạm Văn Đoan
Options

lấy lượt nghe bài hát

parent f3fbe27b
Showing with 347 additions and 572 deletions
app/Console/Commands/CrawlerTrackListenCommand.php 0 → 100644
<?php
namespace App\Console\Commands;
use App\Models\CrawlerListen;
use App\Models\Crontjob;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
class CrawlerTrackListenCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'crawler:get-listen {from_page?}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Lấy lượt nghe của bài hát ở các hệ thống dựa vào tham số truyền vào';
protected $trackRepository;
/**
* Create a new command instance.
*
* CrawlerTrackListenCommand constructor.
* @param TrackRepository $trackRepository
*/
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
$this->trackRepository = $trackRepository;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
// Ghi log xem có gọi không
Crontjob::create(['message' => 'CrawlerTrackListenCommand is called at ' . Carbon::now()]);
$from_page = $this->argument('from_page');
if (empty($from_page) || !in_array($from_page, ['zing', 'nct', 'nhacvn', 'keeng'])) {
$from_page = $this->choice('Chọn trang nhạc ', ['zing', 'nct', 'nhacvn', 'keeng']);
}
echo "\n Bắt đầu xử lý lấy lượt nghe từ: " . $from_page;
$crawlered = $this->trackRepository->getCrawlerListenInserted($from_page, false);
echo "\n Tổng số bài hát đã lấy lượt nghe hôm nay: " . count($crawlered);
// Lấy mảng bài hát theo từng hệ thống nhạc
$inserted = $this->trackRepository->getTrackBySource($from_page, $crawlered);
echo "\n Tổng số bài hát cần lấy lượt nghe: " . count($inserted);
if (count($inserted) > 0) {
$data_listen = [];
foreach ($inserted as $key => $track) {
$listen = $this->privateGetListen($from_page, $track);
if ($listen > 0) {
$data_listen[$track['id']] = $listen;
}
echo "\n " . ($key + 1) . ". " . $track['title'] . " (Track ID= " . $track['id'] . "): " . $listen;
}
if (count($data_listen) > 0) {
foreach ($data_listen as $track_id => $listen ) {
switch ($from_page) {
case 'zing':
CrawlerListen::updateOrCreate(
['track_id' => $track_id],
['today_zing' => $listen, 'zing_crawler_at' => Carbon::now()]
);
break;
case 'nct':
CrawlerListen::updateOrCreate(
['track_id' => $track_id],
['today_nct' => $listen, 'nct_crawler_at' => Carbon::now()]
);
break;
}
}
}
}
echo "\n Lấy số lượt nghe kết thúc.";
}
/**
* Hàm lấy lượt nghe thông qua api của các trang nhạc
*
* @param $src
* @param $track
* @return int
*/
private function privateGetListen($src, $track)
{
$listen = 0;
switch ($src) {
case 'zing':
$url = getZingUrlGetCounter($track['id_zing']);
$curl = cURL($url);
$response = json_decode($curl);
if (isset($response->err) && $response->err == 0) {
if (isset($response->data) && isset($response->data->total) && $response->data->total > 0) {
$listen = $response->data->total;
}
}
break;
case 'nct':
$url = getNctUrlGetCounter($track['id_nct']);
$curl = cURL($url);
$response = json_decode($curl);
if (isset($response->error_code) && $response->error_code == 0) {
if (isset($response->data) && isset($response->data->songs) && $response->data->songs->{$track['id_nct']} > 0) {
$listen = $response->data->songs->{$track['id_nct']};
}
}
break;
}
return $listen;
}
}
app/Console/Kernel.php
...@@ -2,10 +2,12 @@ ...@@ -2,10 +2,12 @@
namespace App\Console; namespace App\Console;
use App\Console\Commands\CrawlerTrackListenCommand;
use App\Console\Commands\CrontjobCommand; use App\Console\Commands\CrontjobCommand;
use App\Console\Commands\Keeng\KeengCrawlerTrackCommand; use App\Console\Commands\Keeng\KeengCrawlerTrackCommand;
use App\Console\Commands\Nct\NctCrawlerTrackCommand; use App\Console\Commands\Nct\NctCrawlerTrackCommand;
use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand; use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
use App\Console\Commands\Zing\ZingCrawlerListenCommand;
use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand; use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand;
use App\Console\Commands\Zing\ZingCrawlerTrackCommand; use App\Console\Commands\Zing\ZingCrawlerTrackCommand;
use App\Console\Commands\CrawlerTrackImageCommand; use App\Console\Commands\CrawlerTrackImageCommand;
...@@ -28,6 +30,10 @@ class Kernel extends ConsoleKernel ...@@ -28,6 +30,10 @@ class Kernel extends ConsoleKernel
KeengCrawlerTrackCommand::class, KeengCrawlerTrackCommand::class,
CrawlerTrackImageCommand::class, CrawlerTrackImageCommand::class,
//ZingCrawlerListenCommand::class,
CrawlerTrackListenCommand::class
]; ];
/** /**
......
app/Helpers/Constants.php
...@@ -5,6 +5,11 @@ namespace App\Helpers; ...@@ -5,6 +5,11 @@ namespace App\Helpers;
class Constants class Constants
{ {
const TABLE_ARTISTS = 'artists'; const TABLE_ARTISTS = 'artists';
const TABLE_COUNT_VIEWS = 'count_views';
const TABLE_CRAWLER_LISTEN_HISTORIES = 'crawler_listen_histories';
const TABLE_CRAWLER_LISTENS = 'crawler_listens';
const TABLE_CRONTJOBS = 'crontjobs';
const TABLE_FAILED_JOBS = 'failed_jobs'; const TABLE_FAILED_JOBS = 'failed_jobs';
const TABLE_JOBS = 'jobs'; const TABLE_JOBS = 'jobs';
const TABLE_MIGRATIONS = 'migrations'; const TABLE_MIGRATIONS = 'migrations';
...@@ -19,9 +24,11 @@ class Constants ...@@ -19,9 +24,11 @@ class Constants
const TABLE_PERMISSIONS = 'permissions'; const TABLE_PERMISSIONS = 'permissions';
const TABLE_ROLE_USER = 'role_user'; const TABLE_ROLE_USER = 'role_user';
const TABLE_ROLES = 'roles'; const TABLE_ROLES = 'roles';
const TABLE_TRACK_ARTIST = 'track_artist'; const TABLE_TRACK_ARTIST = 'track_artist';
const TABLE_TRACK_USER = 'track_user'; const TABLE_TRACK_USER = 'track_user';
const TABLE_TRACKS = 'tracks'; const TABLE_TRACKS = 'tracks';
const TABLE_USER_LOGS = 'user_logs'; const TABLE_USER_LOGS = 'user_logs';
const TABLE_USERS = 'users'; const TABLE_USERS = 'users';
...@@ -117,4 +124,6 @@ class Constants ...@@ -117,4 +124,6 @@ class Constants
const VMUSICCHART_IMG_EMPTY_JPG = '/home/aseanvn/public_html/vmusicchart.dcv.vn/uploads/empty-1x1-jpg.jpg'; const VMUSICCHART_IMG_EMPTY_JPG = '/home/aseanvn/public_html/vmusicchart.dcv.vn/uploads/empty-1x1-jpg.jpg';
const VMUSICCHART_IMG_EMPTY_GIF = '/home/aseanvn/public_html/vmusicchart.dcv.vn/uploads/empty-1x1-gif.gif'; const VMUSICCHART_IMG_EMPTY_GIF = '/home/aseanvn/public_html/vmusicchart.dcv.vn/uploads/empty-1x1-gif.gif';
const CRAWLER_LISTEN_LIMIT = 500;
} }
\ No newline at end of file
app/Helpers/Functions.php
<?php <?php
use App\Helpers\Constants;
use Illuminate\Support\Facades\Auth;
if (!function_exists('includeRouteFiles')) {
/**
* Loops through a folder and requires all PHP files
* Searches sub-directories as well.
*
* @param $folder
*/
function includeRouteFiles($folder)
{
try {
$rdi = new recursiveDirectoryIterator($folder);
$it = new recursiveIteratorIterator($rdi);
while ($it->valid()) {
if (!$it->isDot() && $it->isFile() && $it->isReadable() && $it->current()->getExtension() === 'php') {
require $it->key();
}
$it->next();
}
} catch (Exception $e) {
echo $e->getMessage();
}
}
}
if (!function_exists('processCommonResponse')) {
/**
* @param $result
* @param null $data
* @return \Illuminate\Http\JsonResponse
*/
function processCommonResponse($result, $data = null)
{
return response()->json(array(
'code' => $result ? Constants::CODE_SUCCESS : Constants::CODE_ERROR,
'message' => $result ? Constants::MESSAGE_SUCCESS : Constants::MESSAGE_ERROR,
'data' => $data
));
}
}
if (!function_exists('generateFilenameUpload')) {
/**
* Get filename from file when upload
* @param $file
* @return string
*/
function generateFilenameUpload($file)
{
if (is_object($file) && is_file($file)) {
return 'user_' . Auth::id().'_upload_'.time().'.'.$file->getClientOriginalExtension();
}
return 'user_' . Auth::id().'_upload_'.time();
}
}
if (!function_exists('isAdmin')) {
/**
* check is admin
* @return true,false
*/
function isAdmin() {
$query = \App\User::where('id',Auth::Id());
$query->with([
'roles' => function($roleSql){
$roleSql->select( 'roles.id','roles.name');
}
]);
$result = $query->get()->first();
$role = array();
if(!empty($result) && count($result->roles) > 0){
foreach($result->roles as $val){
$role[] = $val->name;
}
}
if(in_array('administrator', $role) && Auth::check()){
return true;
}
return false;
}
}
/**
* The function check item and return value
*/
if (!function_exists('getArrayItem')) {
/**
* Get filename from file when upload
* @param $array
* @param $key
* @return string
*/
function getArrayItem($array, $key)
{
if(is_array($array) && isset($array[$key])) {
return $array[$key];
} else {
return null;
}
}
}
if (!function_exists('translateKeyWord')) { if (!function_exists('translateKeyWord')) {
/** /**
* Translate special key work * Translate special key work
...@@ -125,248 +16,6 @@ if (!function_exists('translateKeyWord')) { ...@@ -125,248 +16,6 @@ if (!function_exists('translateKeyWord')) {
} }
} }
if (!function_exists('get_browser_name')) {
/**
* Get browser name from user agent
*
* @param $user_agent
* @return string
*/
function get_browser_name($user_agent)
{
if (strpos($user_agent, 'Opera') || strpos($user_agent, 'OPR/')) return 'Opera';
elseif (strpos($user_agent, 'Edge')) return 'edge';
elseif (strpos($user_agent, 'Chrome')) return 'chrome';
elseif (strpos($user_agent, 'Safari')) return 'safari';
elseif (strpos($user_agent, 'Firefox')) return 'firefox';
elseif (strpos($user_agent, 'MSIE') || strpos($user_agent, 'Trident/7')) return 'ie';
return 'Other';
}
}
if (!function_exists('convertFileEncodingToUTF8')) {
/**
* Convert file (csv) encoding to uft-8
*
* @param $filePath
* @return mixed
*/
function convertFileEncodingToUTF8($filePath)
{
try{
if(\File::exists($filePath)){
$inputContent = file_get_contents($filePath);
$inputEncoding = mb_detect_encoding($inputContent, \App\Constants::SPECIFY_ENCODING_LIS);
if (strtoupper($inputEncoding) == 'JIS') {
$inputEncoding = 'SJIS';
}
if (! empty($inputEncoding)) {
if (strtoupper($inputEncoding) != \App\Constants::OUT_CHARSET) {
file_put_contents($filePath, iconv($inputEncoding, \App\Constants::OUT_CHARSET, $inputContent));
}
return true;
} else {
return false;
}
} else {
return false;
}
} catch (Exception $e){
error_log('import encoding: '.$e->getMessage());
return false;
}
}
}
if (!function_exists('checkArrayNull')) {
/**
* Check array null (all item null)
*
* @param array $array
* @return bool
*/
function checkArrayNull(array $array)
{
if (is_array($array) && count($array) > 0) {
$arrayNull = array_filter($array, 'is_null');
if (count($array) === count($arrayNull)) {
return true;
} else {
return false;
}
} else {
return false;
}
}
}
if (!function_exists('isJson')) {
/**
* Check JSON format is valid
*
* @param $string
* @return bool
*/
function isJson($string)
{
json_decode($string);
return (json_last_error() == JSON_ERROR_NONE);
}
}
if (!function_exists('getCustomFieldForExport')) {
/**
* Get custom field for export csv
*
* @param $customFieldDataAll
* @param $receiverCustomField
* @return array
*/
function getCustomFieldForExport($customFieldDataAll, $receiverCustomField, $isSimpleArray=true)
{
if (is_array($customFieldDataAll) && count($customFieldDataAll) > 0) {
if (! empty($receiverCustomField)) {
$customFields = explode('||', $receiverCustomField);
} else {
return [];
}
if (count($customFields) > 1) {
unset($customFields[count($customFields)-1]);
}
$data = [];
foreach ($customFieldDataAll as $customValue) {
$init = null;
foreach ($customFields as $value) {
preg_match("/^\[(.*):=\](.*)$/", $value, $temp);
if (count($temp) == 3) {
unset($temp[0]);
}
if ($customValue['id'] == $temp[1]) {
$init = trim($temp[2]);
}
}
if (empty($init)) {
if ($isSimpleArray) {
$data[][$customValue['id']] = null;
} else {
$data[] = null;
}
} else {
if ($isSimpleArray) {
$data[][$customValue['id']] = $init;
} else {
$data[] = $init;
}
}
}
return $data;
}
return [];
}
}
if (!function_exists('getRoleList')) {
/**
* Get custom field for export csv
* Default call: getRoleList()
*
* @param $userId
* @param $isCurrentUser
* @param $isArray
*
* @return array
*/
function getRoleList($userId=null, $isCurrentUser=true, $isArray=false)
{
if ($isCurrentUser) {
$userId = \auth()->id();
}
$user = \App\Models\User::find($userId);
if ($user) {
$arr = collect($user->roles)->map(function ($item) {
return $item->name;
})->all();
return ($isArray) ? $arr : implode(",", $arr);
} else {
return ($isArray) ? [] : '';
}
}
}
if (!function_exists('getUserListInPartner')) {
/**
* Get custom field for export csv
* Default call: getUserListInPartner()
*
* @param $userId
* @param $isCurrentUser
* @param $isObject
*
* @return array
*/
function getUserListInPartner($userId=null, $isCurrentUser=true, $isObject=false)
{
if ($isCurrentUser) {
$userId = \auth()->id();
}
$user = \App\Models\User::find($userId);
if ($user) {
$users = User::select('id', 'partner_id', 'name', 'email')
->where('partner_id', $user->partner_id)
->get();
if ($isObject) {
return $users;
} else {
return collect($users)->map(function ($item) {
return $item->id;
})->all();
}
} else {
return [];
}
}
}
if (!function_exists('getBirthdayEmailTemplateFromFile')) {
/**
* Hàm lấy template từ file html gửi mail chúc mừng sinh nhật
*
*/
function getBirthdayEmailTemplateFromFile()
{
try {
$path_file = public_path(Constants::BIRTHDAY_EMAIL_TEMPLATE_FROM_PUBLIC . Constants::BIRTHDAY_EMAIL_TEMPLATE_FILENAME);
if (\File::exists($path_file)) {
$email_template_path = storage_path(Constants::BIRTHDAY_EMAIL_TEMPLATE_FROM_STORAGE . Constants::BIRTHDAY_EMAIL_TEMPLATE_FILENAME);
$email_template_file = fopen($email_template_path, 'r');
$email_template_data = fread($email_template_file, filesize($email_template_path));
fclose($email_template_file);
return $email_template_data;
}
return null;
} catch (Exception $e) {
return null;
}
}
}
if (!function_exists('sendRequest')) { if (!function_exists('sendRequest')) {
function sendRequest($url, $params = array(), $method = 'POST', $isJSON = true, $isAuthen = false, $timeOut = 60) function sendRequest($url, $params = array(), $method = 'POST', $isJSON = true, $isAuthen = false, $timeOut = 60)
{ {
...@@ -413,46 +62,6 @@ if (!function_exists('sendRequest')) { ...@@ -413,46 +62,6 @@ if (!function_exists('sendRequest')) {
} }
} }
if (!function_exists('dataResponse')) {
function dataResponse($code, $message, $data) {
return [
'code' => $code,
'message' => $message,
'data' => $data
];
}
}
if (!function_exists('checkTaxCode')) {
function checkTaxCode($taxcode) {
$vn_taxcode = '/^[0-9]{10}$|^[0-9]{10}-[0-9]{2}[1-9]{1}$/';
if (preg_match($vn_taxcode, $taxcode)) {
$prefix_input = substr($taxcode, 0, 3);
$array_prefix = config('validator.tax_code_by_province');
if (count($array_prefix) > 0 && in_array($prefix_input, $array_prefix)) {
$tax_code = substr($taxcode, 0, 10);
if (strlen($tax_code) == 10) {
$total = 31*$tax_code[0];
$total += 29*$tax_code[1];
$total += 23*$tax_code[2];
$total += 19*$tax_code[3];
$total += 17*$tax_code[4];
$total += 13*$tax_code[5];
$total += 7*$tax_code[6];
$total += 5*$tax_code[7];
$total += 3*$tax_code[8];
if ($tax_code[9] == (10 - ($total % 11))) {
return true;
}
return false;
}
return false;
}
return false;
}
}
}
if (!function_exists('getDom')) { if (!function_exists('getDom')) {
function getDom($link) function getDom($link)
{ {
...@@ -466,186 +75,6 @@ if (!function_exists('getDom')) { ...@@ -466,186 +75,6 @@ if (!function_exists('getDom')) {
} }
} }
if (!function_exists('getNotificationStatus')) {
function getNotificationStatus($name, $status) {
$badge = $name . ' ';
switch ($status) {
case 0:
$badge .= '<span class="badge badge-secondary" style="float: right">Chưa đọc</span>';
break;
case 1:
$badge .= '<span class="badge badge-primary" style="float: right">Đã đọc</span>';
break;
case 2:
$badge .= '<span class="badge badge-danger" style="float: right">Đã xóa</span>';
break;
default:
}
return $badge;
}
}
if (!function_exists('getProjectStatus')) {
function getProjectStatus($name, $status) {
$badge = $name . ' ';
switch ($status) {
case 0:
$badge .= '<span class="badge badge-secondary" style="float: right">Bản nháp</span>';
break;
case 1:
$badge .= '<span class="badge badge-info" style="float: right">Chờ duyệt</span>';
break;
case 2:
$badge .= '<span class="badge badge-primary" style="float: right">Đã duyệt</span>';
break;
case 3:
$badge .= '<span class="badge badge-warning" style="float: right">Đang thực hiện</span>';
break;
case 4:
$badge .= '<span class="badge badge-success" style="float: right">Hoàn thành</span>';
break;
case 5:
$badge .= '<span class="badge badge-danger" style="float: right">Tạm dừng</span>';
break;
default:
}
return $badge;
}
}
if (!function_exists('getTaskStatus')) {
function getTaskStatus($name, $status) {
$badge = $name . ' ';
switch ($status) {
case 0:
$badge .= '<span class="badge badge-secondary" style="float: right">Bản nháp</span>';
break;
case 1:
$badge .= '<span class="badge badge-info" style="float: right">Đã duyệt</span>';
break;
case 2:
$badge .= '<span class="badge badge-primary" style="float: right">Đã giao</span>';
break;
case 3:
$badge .= '<span class="badge badge-warning" style="float: right">Đang thực hiện</span>';
break;
case 4:
$badge .= '<span class="badge badge-success" style="float: right">Hoàn thành</span>';
break;
case 5:
$badge .= '<span class="badge badge-danger" style="float: right">Tạm dừng</span>';
break;
case 6:
$badge .= '<span class="badge badge-dark" style="float: right">Chuyển tiếp</span>';
break;
default:
}
return $badge;
}
}
if (!function_exists('getUserStatus')) {
function getUserStatus($name, $status) {
$badge = $name . ' ';
switch ($status) {
case 0:
$badge .= '<span class="badge badge-secondary" style="float: right">Chưa kích hoạt</span>';
break;
case 1:
$badge .= '<span class="badge badge-info" style="float: right">Đã kích hoạt</span>';
break;
case 2:
$badge .= '<span class="badge badge-warning" style="float: right">Tạm khóa</span>';
break;
case 3:
$badge .= '<span class="badge badge-danger" style="float: right">Đã nghỉ làm</span>';
break;
default:
}
return $badge;
}
}
if (!function_exists('validateMobile')) {
function validateMobile($mobile) {
if ($mobile == "" || $mobile == null) return false;
$start_pattern = "/";
$end_pattern = "/";
$viettel_pattern = $start_pattern;
$viettel_pattern .= "^8498\d{7}$|^0?98\d{7}$|^98\d{7}$";
$viettel_pattern .= "|^8497\d{7}$|^0?97\d{7}$|^97\d{7}$";
$viettel_pattern .= "|^8496\d{7}$|^0?96\d{7}$|^96\d{7}$";
$viettel_pattern .= "|^8432\d{7}$|^0?32\d{7}$|^32\d{7}$";
$viettel_pattern .= "|^8433\d{7}$|^0?33\d{7}$|^33\d{7}$";
$viettel_pattern .= "|^8434\d{7}$|^0?34\d{7}$|^34\d{7}$";
$viettel_pattern .= "|^8435\d{7}$|^0?35\d{7}$|^35\d{7}$";
$viettel_pattern .= "|^8436\d{7}$|^0?36\d{7}$|^36\d{7}$";
$viettel_pattern .= "|^8437\d{7}$|^0?37\d{7}$|^37\d{7}$";
$viettel_pattern .= "|^8438\d{7}$|^0?38\d{7}$|^38\d{7}$";
$viettel_pattern .= "|^8439\d{7}$|^0?39\d{7}$|^39\d{7}$";
$viettel_pattern .= "|^8486\d{7}$|^0?86\d{7}$|^86\d{7}$";
$vinaphone_pattern = $viettel_pattern;
$vinaphone_pattern .= "|^8491\d{7}$|^0?91\d{7}$|^91\d{7}$";
$vinaphone_pattern .= "|^8494\d{7}$|^0?94\d{7}$|^94\d{7}$";
$vinaphone_pattern .= "|^8481\d{7}$|^0?81\d{7}$|^81\d{7}$";
$vinaphone_pattern .= "|^8482\d{7}$|^0?82\d{7}$|^82\d{7}$";
$vinaphone_pattern .= "|^8483\d{7}$|^0?83\d{7}$|^83\d{7}$";
$vinaphone_pattern .= "|^8484\d{7}$|^0?84\d{7}$|^84\d{7}$";
$vinaphone_pattern .= "|^8485\d{7}$|^0?85\d{7}$|^85\d{7}$";
$vinaphone_pattern .= "|^8488\d{7}$|^0?88\d{7}$|^88\d{7}$";
$mobifone_pattern = $vinaphone_pattern;
$mobifone_pattern .= "|^8490\d{7}$|^0?90\d{7}$|^90\d{7}$";
$mobifone_pattern .= "|^8493\d{7}$|^0?93\d{7}$|^93\d{7}$";
$mobifone_pattern .= "|^8470\d{7}$|^0?70\d{7}$|^70\d{7}$";
$mobifone_pattern .= "|^8476\d{7}$|^0?76\d{7}$|^76\d{7}$";
$mobifone_pattern .= "|^8477\d{7}$|^0?77\d{7}$|^77\d{7}$";
$mobifone_pattern .= "|^8478\d{7}$|^0?78\d{7}$|^78\d{7}$";
$mobifone_pattern .= "|^8479\d{7}$|^0?79\d{7}$|^79\d{7}$";
$mobifone_pattern .= "|^8489\d{7}$|^0?89\d{7}$|^89\d{7}$";
$vietnamobile_pattern = $mobifone_pattern;
$vietnamobile_pattern .= "|^8492\d{7}$|^0?92\d{7}$|^92\d{7}$";
$vietnamobile_pattern .= "|^8456\d{7}$|^0?56\d{7}$|^56\d{7}$";
$vietnamobile_pattern .= "|^8458\d{7}$|^0?58\d{7}$|^58\d{7}$";
$vietnamobile_pattern .= $end_pattern;
// $landline_pattern = /^84203\d{8}$|^0?203\d{8}$|^203\d{8}$|^84204\d{8}$|^0?204\d{8}$|^204\d{8}$|^84205\d{8}$|^0?205\d{8}$|^205\d{8}$|^84206\d{8}$|^0?206\d{8}$|^206\d{8}$|^84207\d{8}$|^0?207\d{8}$|^207\d{8}$|^84208\d{8}$|^0?208\d{8}$|^208\d{8}$|^84209\d{8}$|^0?209\d{8}$|^209\d{8}$|^84210\d{8}$|^0?210\d{8}$|^210\d{8}$|^84211\d{8}$|^0?211\d{8}$|^211\d{8}$|^84212\d{8}$|^0?212\d{8}$|^212\d{8}$|^84213\d{8}$|^0?213\d{8}$|^213\d{8}$|^84214\d{8}$|^0?214\d{8}$|^214\d{8}$|^84215\d{8}$|^0?215\d{8}$|^215\d{8}$|^84216\d{8}$|^0?216\d{8}$|^216\d{8}$|^84218\d{8}$|^0?218\d{8}$|^218\d{8}$|^84219\d{8}$|^0?219\d{8}$|^219\d{8}$|^84220\d{8}$|^0?220\d{8}$|^220\d{8}$|^84221\d{8}$|^0?221\d{8}$|^221\d{8}$|^84222\d{8}$|^0?222\d{8}$|^222\d{8}$|^84225\d{8}$|^0?225\d{8}$|^225\d{8}$|^84226\d{8}$|^0?226\d{8}$|^226\d{8}$|^84227\d{8}$|^0?227\d{8}$|^227\d{8}$|^84228\d{8}$|^0?228\d{8}$|^228\d{8}$|^84229\d{8}$|^0?229\d{8}$|^229\d{8}$|^84232\d{8}$|^0?232\d{8}$|^232\d{8}$|^84233\d{8}$|^0?233\d{8}$|^233\d{8}$|^84234\d{8}$|^0?234\d{8}$|^234\d{8}$|^84235\d{8}$|^0?235\d{8}$|^235\d{8}$|^84236\d{8}$|^0?236\d{8}$|^236\d{8}$|^84237\d{8}$|^0?237\d{8}$|^237\d{8}$|^84238\d{8}$|^0?238\d{8}$|^238\d{8}$|^84239\d{8}$|^0?239\d{8}$|^239\d{8}$|^8424\d{8}$|^0?24\d{8}$|^24\d{8}$|^84251\d{8}$|^0?251\d{8}$|^251\d{8}$|^84252\d{8}$|^0?252\d{8}$|^252\d{8}$|^84254\d{8}$|^0?254\d{8}$|^254\d{8}$|^84255\d{8}$|^0?255\d{8}$|^255\d{8}$|^84256\d{8}$|^0?256\d{8}$|^256\d{8}$|^84257\d{8}$|^0?257\d{8}$|^257\d{8}$|^84258\d{8}$|^0?258\d{8}$|^258\d{8}$|^84259\d{8}$|^0?259\d{8}$|^259\d{8}$|^84260\d{8}$|^0?260\d{8}$|^260\d{8}$|^84261\d{8}$|^0?261\d{8}$|^261\d{8}$|^84262\d{8}$|^0?262\d{8}$|^262\d{8}$|^84263\d{8}$|^0?263\d{8}$|^263\d{8}$|^84269\d{8}$|^0?269\d{8}$|^269\d{8}$|^84270\d{8}$|^0?270\d{8}$|^270\d{8}$|^84271\d{8}$|^0?271\d{8}$|^271\d{8}$|^84272\d{8}$|^0?272\d{8}$|^272\d{8}$|^84273\d{8}$|^0?273\d{8}$|^273\d{8}$|^84274\d{8}$|^0?274\d{8}$|^274\d{8}$|^84275\d{8}$|^0?275\d{8}$|^275\d{8}$|^84276\d{8}$|^0?276\d{8}$|^276\d{8}$|^84277\d{8}$|^0?277\d{8}$|^277\d{8}$|^8428\d{8}$|^0?28\d{8}$|^28\d{8}$|^84290\d{8}$|^0?290\d{8}$|^290\d{8}$|^84291\d{8}$|^0?291\d{8}$|^291\d{8}$|^84292\d{8}$|^0?292\d{8}$|^292\d{8}$|^84293\d{8}$|^0?293\d{8}$|^293\d{8}$|^84294\d{8}$|^0?294\d{8}$|^294\d{8}$|^84296\d{8}$|^0?296\d{8}$|^296\d{8}$|^84297\d{8}$|^0?297\d{8}$|^297\d{8}$|^84299\d{8}$|^0?299\d{8}$|^299\d{8}$/;
if (preg_match($vietnamobile_pattern, $mobile)) {
return true;
} else{
return false;
}
}
}
if (!function_exists('formatMobile')) {
function formatMobile($mobile) {
$res_format = '';
if (validateMobile($mobile)) {
switch (strlen($mobile)) {
case 9:
$res_format = '84' . $mobile;
break;
case 10:
$res_format = '84' . substr($mobile, 1);
break;
case 11:
$res_format = $mobile;
default:
}
}
return $res_format;
}
}
if (!function_exists('curlZingMp3')) { if (!function_exists('curlZingMp3')) {
function curlZingMp3($url) { function curlZingMp3($url) {
$ch = curl_init(); $ch = curl_init();
...@@ -749,4 +178,52 @@ if (!function_exists('cURL')) { ...@@ -749,4 +178,52 @@ if (!function_exists('cURL')) {
} }
} }
if (!function_exists('getZingUrlGetCounter')) {
/**
* Hàm tạo URL cho từng bài hát dựa vào ID chuỗi của nó để lấy lượt nghe
* https://mp3.zing.vn/xhr/get-counter?id='. $idTrack .'&type=audio
*
* @param $id_str
* @param string $type
* @return string
*/
function getZingUrlGetCounter($id_str, $type='audio') {
$url = 'https://mp3.zing.vn/xhr/get-counter?';
$url .= 'id=' . $id_str;
$url .= '&type=' . $type;
return $url;
}
}
if (!function_exists('getNctUrlGetCounter')) {
/**
* Hàm tạo URL cho từng bài hát dựa vào ID của nó để lấy lượt nghe
* https://www.nhaccuatui.com/interaction/api/counter?listSongIds=6238530
*
* @param $id
* @return string
*/
function getNctUrlGetCounter($id) {
$url = 'https://www.nhaccuatui.com/interaction/api/counter?';
$url .= 'listSongIds=' . $id;
return $url;
}
}
if (!function_exists('getKeengUrlGetCounter')) {
/**
* Hàm tạo URL cho từng bài hát dựa vào ID của nó để lấy lượt nghe
* http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getSong?identify=IGm9Y2XF
*
* @param $id
* @return string
*/
function getKeengUrlGetCounter($id) {
$url = 'http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getSong?';
$url .= 'identify=' . $id;
return $url;
}
}
app/Models/CrawlerListen.php 0 → 100644
<?php
namespace App\Models;
use App\Helpers\Constants;
use Illuminate\Database\Eloquent\Model;
class CrawlerListen extends Model
{
protected $table = Constants::TABLE_CRAWLER_LISTENS;
public $timestamps = true;
protected $fillable = [
'track_id',
'yesterday_zing','yesterday_nct','yesterday_nhacvn','yesterday_keeng',
'today_zing','today_nct','today_nhacvn','today_keeng',
'date_zing','date_nct','date_nhacvn','date_keeng',
'week_zing','week_nct','week_nhacvn','week_keeng',
'month_zing','month_nct','week_nhacvn','month_keeng',
'year_zing','year_nct','year_nhacvn','year_keeng',
'alltime_zing','alltime_nct','alltime_nhacvn','alltime_keeng',
'zing_crawler_at','nct_crawler_at','nhacvn_crawler_at','keeng_crawler_at'
];
}
app/Models/CrawlerListenHistory.php 0 → 100644
<?php
namespace App\Models;
use App\Helpers\Constants;
use Illuminate\Database\Eloquent\Model;
class CrawlerListenHistory extends Model
{
protected $table = Constants::TABLE_CRAWLER_LISTEN_HISTORIES;
public $timestamps = true;
protected $fillable = [
'track_id',
'zing_count',
'nct_count',
'nhacvn_count',
'keeng_count'
];
}
app/Models/Crontjob.php
...@@ -2,11 +2,12 @@ ...@@ -2,11 +2,12 @@
namespace App\Models; namespace App\Models;
use App\Helpers\Constants;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
class Crontjob extends Model class Crontjob extends Model
{ {
protected $table = 'crontjobs'; protected $table = Constants::TABLE_CRONTJOBS;
public $timestamps = true; public $timestamps = true;
......
app/Repositories/TrackRepository.php
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
namespace App\Repositories; namespace App\Repositories;
use App\Helpers\Constants;
use App\Models\CrawlerListen;
use App\Models\Track; use App\Models\Track;
use Sunra\PhpSimple\HtmlDomParser; use Sunra\PhpSimple\HtmlDomParser;
use Yangqi\Htmldom\Htmldom; use Yangqi\Htmldom\Htmldom;
...@@ -279,5 +281,88 @@ class TrackRepository extends BaseRepository ...@@ -279,5 +281,88 @@ class TrackRepository extends BaseRepository
return $result; return $result;
} }
/**
* Lấy ds bài hát chưa lấy lượt nghe trong ngày theo từng hệ thống nhạc
* Bài nào đã lấy rồi thì sẽ ko lấy nữa
* Mỗi lần quét 500 bài cho đến hết
*
* @param string $src
* @param $crawlered
* @param int $limit
*
* @return array
*/
public function getTrackBySource($src='zing', $crawlered, $limit = Constants::CRAWLER_LISTEN_LIMIT)
{
$query = Track::select(['id', 'id_zing', 'id_nct', 'id_nhacvn', 'id_keeng', 'title']);
if (is_array($crawlered) && count($crawlered) > 0) {
$query->whereNotIn('id', $crawlered);
}
switch ($src) {
case 'zing': $query->whereNotNull('id_zing')->where('id_zing', '<>', ''); break;
case 'nct': $query->whereNotNull('id_nct')->where('id_nct', '<>', ''); break;
case 'nhacvn': $query->whereNotNull('id_nhacvn')->where('id_nhacvn', '<>', ''); break;
case 'keeng': $query->whereNotNull('id_keeng')->where('id_keeng', '<>', ''); break;
}
$query->take(($limit < 1) ? Constants::CRAWLER_LISTEN_LIMIT : $limit);
return $query->get()->toArray();
}
/**
* Lấy ds những bài hát đã lấy lượt nghe trong ngày, để lần quét sau sẽ bỏ qua
* Mỗi lần quét 300-500 bài trong tracks và bỏ qua những bài đã lấy lượt nghe trong ngày ở bảng crawler_listens
* Lấy những bài hát có lượng nghe trong ngày = 0 để cập nhật lại
* Có thể tùy biến thêm ($is_update_all): cập nhật toàn bộ hoặc chỉ cập nhập những bài chưa lấy lượt nghe
*
* @param string $src
* @param boolean $is_update_all
*
* @return array
*/
public function getCrawlerListenInserted($src='zing', $is_update_all=false)
{
$min_date = date('Y-m-d 00:00:00');
$max_date = date('Y-m-d 23:59:59');
if ($is_update_all) {
$min_date = date('Y-m-d 00:00:00', time() + 86400);
$max_date = date('Y-m-d 23:59:59', time() + 86400);
}
$query = CrawlerListen::select(['id', 'track_id']);
switch ($src) {
case 'zing':
$query->where('zing_crawler_at', '>=', $min_date)
->where('zing_crawler_at', '<=', $max_date)
->where('today_zing', '>' ,0);
break;
case 'nct':
$query->where('nct_crawler_at', '>=', $min_date)
->where('nct_crawler_at', '<=', $max_date)
->where('today_nct', '>' ,0);
break;
case 'nhacvn':
$query->where('nhacvn_crawler_at', '>=', $min_date)
->where('nhacvn_crawler_at', '<=', $max_date)
->where('today_nhacvn', '>' ,0);
break;
case 'keeng':
$query->where('keeng_crawler_at', '>=', $min_date)
->where('keeng_crawler_at', '<=', $max_date)
->where('today_keeng', '>' ,0);
break;
}
$result = $query->get();
return collect($result)->map(function ($item) { return $item->track_id; })->all();
}
} }
\ No newline at end of file
config/api.php
...@@ -85,6 +85,13 @@ return [ ...@@ -85,6 +85,13 @@ return [
'api_get_rank' => 'http://vip.service.keeng.vn:8080/KeengWSRestful//ws/common/getRankDetail?item_type=1&rank_type=50' 'api_get_rank' => 'http://vip.service.keeng.vn:8080/KeengWSRestful//ws/common/getRankDetail?item_type=1&rank_type=50'
], ],
'listen' => [
'zing' => 'https://mp3.zing.vn/xhr/get-counter?id=ZWB0EF7E&type=audio',
'nct' => 'https://www.nhaccuatui.com/interaction/api/counter?listSongIds=6238530',
'nhacvn' => null,
'keeng' => null
],
]; ];
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment