Commit 7da03848 by doan281

Merge branch 'master' into dev

# Conflicts:
#	app/Console/Commands/CrawlerNctTrackListenCommand.php
#	app/Repositories/TrackRepository.php
#	composer.json
parents 4a921f13 bbcb095e
APP_NAME=Laravel
APP_ENV=production
APP_ENV=localhost
#APP_ENV=production/localhost
APP_KEY=base64:OnxDnOd5hyBzqzaoj1c+yl17+9lQiallMbwKuitg4XU=
APP_DEBUG=true
......@@ -41,3 +41,6 @@ MAIL_FROM_NAME=VMusicChart.Vn
PUSHER_APP_ID=
PUSHER_APP_KEY=
PUSHER_APP_SECRET=
# YOUTUBE_API_KEY=AIzaSyAEPc-G8kluV2MZinW6CzP24T9Jk_In2aw
YOUTUBE_API_KEY=AIzaSyAoiNPZC26cXgNAz-s7Shqav5KuIL0BZ_8
......@@ -77,7 +77,7 @@ class CalculateReportCommand extends Command
echo "\n - BXH Ngày đã được tạo trước đó.";
} else {
echo "\n - Bắt đầu xử lý BXH Ngày...";
$daily_data = DailyReportView::select('*')->take(105)->get()->toArray();
$daily_data = DailyReportView::select('*')->take(120)->get()->toArray();
if (count($daily_data) > 0) {
$daily_report = [];
foreach ($daily_data as $key => $daily) {
......@@ -165,7 +165,7 @@ class CalculateReportCommand extends Command
if($check >= 100){
echo "\n - BXH Tuần đã được tổng hợp trước đó.";
} else {
$weekly_data = WeeklyReportView::select('*')->take(105)->get()->toArray();
$weekly_data = WeeklyReportView::select('*')->take(120)->get()->toArray();
if (count($weekly_data) > 0 && $weekly_data[0]['total_point'] > 0) {
echo "\n - Bắt đầu xử lý BXH Tuần...";
$weekly_report = [];
......@@ -235,7 +235,7 @@ class CalculateReportCommand extends Command
if($check >= 100){
echo "\n - BXH Tháng đã được tổng hợp trước đó.";
} else {
$monthly_data = MonthlyReportView::select('*')->take(105)->get()->toArray();
$monthly_data = MonthlyReportView::select('*')->take(120)->get()->toArray();
if (count($monthly_data) > 0 && $monthly_data[0]['total_point'] > 0) {
echo "\n - Bắt đầu xử lý BXH Tháng...";
$monthly_report = [];
......
......@@ -66,8 +66,11 @@ class CrawlerTrackListenCommand extends Command
echo "\n Tổng số bài hát đã lấy lượt nghe hôm nay: " . count($crawlered);
// Lấy blacklist
$blacklist_listen = $this->trackRepository->getBacklistListen($from_page);
// Lấy mảng bài hát theo từng hệ thống nhạc
$inserted = $this->trackRepository->getTrackBySource($from_page, $crawlered, $limit);
$inserted = $this->trackRepository->getTrackBySource($from_page, $crawlered, $limit, false, $blacklist_listen);
$total = count($inserted);
echo "\n Tổng số bài hát cần lấy lượt nghe: " . $total;
......@@ -139,7 +142,7 @@ class CrawlerTrackListenCommand extends Command
break;
case 'nct':
$url = getNctUrlGetCounter($track['id_nct']);
/*$url = getNctUrlGetCounter($track['id_nct']);
$curl = cURL($url);
$response = json_decode($curl);
......@@ -147,6 +150,28 @@ class CrawlerTrackListenCommand extends Command
if (isset($response->data) && isset($response->data->songs) && $response->data->songs->{$track['id_nct']} > 0) {
$listen = $response->data->songs->{$track['id_nct']};
}
}*/
/* Tính lượt nghe lấy data từ biểu đồ lượt nghe */
$url = getNctUrlGetCounterForChart($track['id_nct']);
$curl = cURL($url);
$response = json_decode($curl);
if (isset($response->status) && $response->status == 'Success') {
if (isset($response->dailyChart) && isset($response->dateChart) && isset($response->cumuChart) &&
is_array($response->dailyChart) && is_array($response->dateChart) && is_array($response->cumuChart))
{
/*foreach ($response->dailyChart as $key => $val) {
$listen[$key]['track_id'] = $track['id'];
$listen[$key]['daily_chart'] = $val;
$listen[$key]['date_chart'] = date('Y-m-d', strtotime(str_replace('/', '-', $response->dateChart[$key])));
$listen[$key]['cumu_chart'] = $response->cumuChart[$key];
}*/
//echo "\n " . end($response->cumuChart) . " | " . end($response->dailyChart) . " : " . $track['id_nct'];
$listen = end($response->cumuChart) + end($response->dailyChart);
}
}
break;
......
<?php
namespace App\Console\Commands\Gameshow;
use App\Models\Gameshow\Answer;
use App\Models\Gameshow\Question;
use App\Models\Gameshow\Topic;
use Illuminate\Console\Command;
class QuestionCommand extends Command
{
protected $question;
protected $topic;
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'gameshow:question';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle(Question $question, Topic $topic)
{
$this->question = $question;
$this->topic = $topic;
$query = Question::select()
->with([
'topics' => function($sql) {
//$sql->select(['topics.id','topics.name'])->take(2);
$sql->select(['topics.id','topics.name']);
}
])
->withCount('topics')
->get()
->toArray();
print_r($query);
$result = collect($query)->map(function ($item){
})->all();
$q = Question::select('*')
->with([
'answers' => function($sql){
$sql->select(['answers.id', 'answers.name']);
}
])
->get()
->toArray();
print_r($q);
/*$as = Answer::select('*')->with([
'question' => function($sql){
$sql->select(['id', 'name']);
}
])->get()->toArray();
print_r($as);*/
}
}
......@@ -57,12 +57,17 @@ class KeengCrawlerNewRealeaseCommand extends Command
$domain = config('api.keeng.domain');
$tracks = [];
$index = 0;
for ($i = 1; $i <= 10; $i++) {
for ($i = 1; $i <= 1; $i++) {
$url = getKeengUrlNewRealease($i);
$curl = cURL($url);
$data = json_decode($curl);
if (isset($data->data)) {
foreach ($data->data as $key => $track) {
$id_keeng = (isset($track->identify) && $track->identify != '') ? $track->identify : null;
if (empty($id_keeng)) {
$id_keeng = (isset($track->info_extra->identify)) ? $track->info_extra->identify : null;
}
//
$tracks[$index]['title'] = isset($track->name) ? $track->name : 'unknown-title';
$tracks[$index]['slug'] = isset($track->slug) ? $track->slug : null;
$tracks[$index]['userid'] = 0;
......@@ -71,7 +76,7 @@ class KeengCrawlerNewRealeaseCommand extends Command
$tracks[$index]['download_hash'] = md5($track->id);
$tracks[$index]['time'] = time();
$tracks[$index]['link_keeng'] = isset($track->url) ? $track->url : null;
$tracks[$index]['id_keeng'] = isset($track->identify) ? $track->identify : null;
$tracks[$index]['id_keeng'] = $id_keeng;
$tracks[$index]['singer_list'] = isset($track->singer) ? $track->singer : null;
$tracks[$index]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
$tracks[$index]['src_thumbnail'] = isset($track->image) ? $track->image : null;
......
......@@ -71,7 +71,12 @@ class KeengCrawlerTrackCommand extends Command
$track_db = Track::where('title', $track->name)->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->link_keeng = isset($track->url) ? $track->url : null;
$track_db[0]->id_keeng = isset($track->identify) ? $track->identify : null;
// ID keeng nằm ở 2 key data khác nhau, có bài chỉ ở 1 nơi nên phải check
$id_keeng = (isset($track->identify) && $track->identify != '') ? $track->identify : null;
if (empty($id_keeng)) {
$id_keeng = (isset($track->info_extra->identify)) ? $track->info_extra->identify : null;
}
$track_db[0]->id_keeng = $id_keeng;
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = isset($track->singer) ? $track->singer : null;
}
......
......@@ -169,7 +169,7 @@ class UpdateTrackArtistCommand extends Command
}
// Xử lý cập nhật nhạc sĩ ------------------------------------------------------------------------------
$tracks_nct = $this->trackRepository->getTrackBySource($src, null, 100, true);
$tracks_nct = $this->trackRepository->getTrackBySource($src, null, 100, true, null);
$track_id_composer = $this->trackRepository->getComposerBySource($src, $tracks_nct);
if (count($track_id_composer) > 0) {
$this->privateUpdateTrackSinger($src, $tracks_nct, [], $track_id_composer);
......
<?php
namespace App\Console\Commands\Ynv;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class YnvCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'ynv:get-track {page_index?}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$page_index = $this->argument('page_index');
$page_index = intval($page_index);
echo "\n Starting...";
$results = [];
$domain = 'http://www.yeunhacvang.com/';
//$page_index = 34;
//for ($page = 1; $page <= 140; $page++) {
for ($page = $page_index; $page <= $page_index; $page++) {
echo "\n - Processing page " . $page;
$title = [];
$link = [];
$singer = [];
$url = $domain . '?pg=list&page='.$page;
$dom = getDom($url);
/* Lấy tên bài hát */
$element_track_title = 'div.main table#table1 tr td a';
foreach ($dom->find($element_track_title) as $data) {
$title[] = str_replace([], [], $data->innertext);
}
/* Lấy link bài hát */
$element_track_title = 'div.main table#table1 tr td a';
foreach ($dom->find($element_track_title) as $data) {
$link[] = $domain . str_replace([], [], $data->href);
}
/* Lấy tên ca sĩ */
$element_track_title = 'div.main table#table1 tr td';
foreach ($dom->find($element_track_title) as $data) {
if ($data->width == "40%") {
$singer[] = str_replace([], [], $data->innertext);
}
}
echo "\n -> Số bài " . count($title);
for ($i=0; $i < count($title); $i++) {
preg_match("/-([0-9]+)$/", $link[$i], $ids);
$results[] = [
'page_index' => $page_index,
'track_id' => (is_array($ids) && count($ids) == 2) ? $ids[1] : null,
'title' => $title[$i],
'singer' => $singer[$i+1],
'detail_link' => $link[$i],
'is_checking' => 0,
'created_at' => Carbon::now(),
'updated_at' => Carbon::now()
];
}
}
//print_r($results);
if (count($results) > 0) {
DB::table('ynvs')->insert($results);
}
echo "\n Done.";
}
}
<?php
namespace App\Console\Commands\Youtube;
use Alaouy\Youtube\Facades\Youtube;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Hash;
use Yangqi\Htmldom\Htmldom;
class YoutubeCrawlerListenCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'get:youtube-listen';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
ini_set('max_execution_time', -1);
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
//$url = 'https://www.youtube.com/watch?v=8mC7oNkz9t4';
$video = Youtube::getVideoInfo('UqKVL56IJB8');
$data = [
'id' => $video->id,
'published_at' => date('Y-m-d H:I:s', strtotime($video->snippet->publishedAt)),
'title' => $video->snippet->title,
'thumbnails' => $video->snippet->thumbnails->standard->url,
'duration' => $video->contentDetails->duration,
'view_count' => $video->statistics->viewCount,
];
print_r($data);
//echo Hash::make('123456a@A');
}
}
......@@ -4,7 +4,6 @@ namespace App\Console;
use App\Console\Commands\CalculateDailyListenCommand;
use App\Console\Commands\CalculateReportCommand;
use App\Console\Commands\CrawlerNctTrackListenCommand;
use App\Console\Commands\CrawlerTrackListenCommand;
use App\Console\Commands\CrontjobCommand;
use App\Console\Commands\ExportRankingCommand;
......@@ -15,6 +14,7 @@ use App\Console\Commands\Nct\NctCrawlerTrackCommand;
use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
use App\Console\Commands\SendWarningCommand;
use App\Console\Commands\Update\UpdateTrackArtistCommand;
use App\Console\Commands\Youtube\YoutubeCrawlerListenCommand;
use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand;
use App\Console\Commands\Zing\ZingCrawlerTrackCommand;
use App\Console\Commands\CrawlerTrackImageCommand;
......@@ -30,6 +30,7 @@ class Kernel extends ConsoleKernel
*/
protected $commands = [
CrontjobCommand::class,
YoutubeCrawlerListenCommand::class,
ZingCrawlerTrackCommand::class,
ZingCrawlerNewRealeaseCommand::class,
......@@ -52,9 +53,6 @@ class Kernel extends ConsoleKernel
SendWarningCommand::class,
CrawlerNctTrackListenCommand::class,
];
/**
......@@ -72,6 +70,9 @@ class Kernel extends ConsoleKernel
$schedule->command('nct:get-track')->dailyAt('13:10');
//$schedule->command('nhacvn:get-track')->dailyAt('13:20');
$schedule->command('keeng:get-track')->dailyAt('13:30');
$schedule->command('keeng:get-new-release-track')->dailyAt('13:35');
// Cập nhật ảnh bài hát
$schedule->command('crawler:get-track-image')->dailyAt('13:40');
// Cập nhật thông tin ca sĩ, nhạc sĩ sau 21h
......
......@@ -6,7 +6,9 @@ class Constants
{
const TABLE_ARTISTS = 'artists';
const TABLE_COUNT_VIEWS = 'count_views';
const TABLE_CRAWLER_LISTEN_BLACKLISTS = 'crawler_listen_blacklists';
const TABLE_CRAWLER_LISTEN_HISTORIES = 'crawler_listen_histories';
const TABLE_CRAWLER_LISTEN_NCTS = 'crawler_listen_ncts';
const TABLE_CRAWLER_LISTENS = 'crawler_listens';
const TABLE_CRONTJOBS = 'crontjobs';
......
......@@ -245,6 +245,22 @@ if (!function_exists('getNctUrlGetCounter')) {
}
}
if (!function_exists('getNctUrlGetCounterForChart')) {
/**
* Hàm tạo URL cho từng bài hát dựa vào ID của nó để lấy lượt nghe
* https://www.nhaccuatui.com/ajax/chart?songId=6290700
*
* @param $id
* @return string
*/
function getNctUrlGetCounterForChart($id) {
$url = 'https://www.nhaccuatui.com/ajax/chart?';
$url .= 'songId=' . $id;
return $url;
}
}
if (!function_exists('getKeengUrlGetCounter')) {
/**
* Hàm tạo URL cho từng bài hát dựa vào ID của nó để lấy lượt nghe
......
<?php
namespace App\Models;
use App\Helpers\Constants;
use Illuminate\Database\Eloquent\Model;
class CrawlerListenBlacklist extends Model
{
protected $table = Constants::TABLE_CRAWLER_LISTEN_BLACKLISTS;
public $timestamps = true;
protected $fillable = [
'track_id', 'src', 'title'
];
}
<?php
namespace App\Models;
use App\Helpers\Constants;
use Illuminate\Database\Eloquent\Model;
class CrawlerListenNct extends Model
{
protected $table = Constants::TABLE_CRAWLER_LISTEN_NCTS;
public $timestamps = true;
protected $fillable = [
'track_id',
'date_chart',
'cumu_chart',
'daily_chart',
'is_get_reporting'
];
}
<?php
namespace App\Models\Gameshow;
use Illuminate\Database\Eloquent\Model;
class Answer extends Model
{
protected $table = 'answers';
public function question()
{
return $this->belongsTo('App\Models\Gameshow\Question','question_id','id');
}
}
<?php
namespace App\Models\Gameshow;
use Illuminate\Database\Eloquent\Model;
class Question extends Model
{
protected $table = 'questions';
public function topics()
{
return $this->belongsToMany(
'App\Models\Gameshow\Topic',
'question_topic',
'question_id',
'topic_id'
);
}
public function answers()
{
return $this->hasMany('App\Models\Gameshow\Answer', 'question_id', 'id');
}
}
<?php
namespace App\Models\Gameshow;
use Illuminate\Database\Eloquent\Model;
class Topic extends Model
{
protected $table = 'topics';
public function topics()
{
return $this->belongsToMany(
'App\Models\Gameshow\Question',
'question_topic',
'topic_id',
'question_id'
);
}
}
<?php
namespace App\Models\Ynv;
use Illuminate\Database\Eloquent\Model;
class Ynv extends Model
{
protected $table = 'ynvs';
public $timestamps = true;
protected $fillable = [
'page_index',
'track_id',
'title',
'singer',
'detail_link',
'mp3_link',
'lyric',
'is_checking'
];
}
......@@ -4,6 +4,7 @@ namespace App\Repositories;
use App\Helpers\Constants;
use App\Models\CrawlerListen;
use App\Models\CrawlerListenBlacklist;
use App\Models\Track;
use Goutte\Client;
use Sunra\PhpSimple\HtmlDomParser;
......@@ -293,6 +294,8 @@ class TrackRepository extends BaseRepository
* Lấy ds bài hát chưa lấy lượt nghe trong ngày theo từng hệ thống nhạc
* Bài nào đã lấy rồi thì sẽ ko lấy nữa
* Mỗi lần quét 500 bài cho đến hết
* Các bài không lấy ở Zing: 1139,3835,3664
* Các bài không lấy ở NCT: 1198,1700,1954,3223,3334,3357,3529,3688,3918,3664,3910
*
* @param string $src
* @param $crawlered
......@@ -300,7 +303,7 @@ class TrackRepository extends BaseRepository
*
* @return array
*/
public function getTrackBySource($src='zing', $crawlered, $limit = Constants::CRAWLER_LISTEN_LIMIT, $is_update_composer=false)
public function getTrackBySource($src='zing', $crawlered, $limit = Constants::CRAWLER_LISTEN_LIMIT, $is_update_composer=false, $blacklist_listen=null)
{
$query = Track::select(['id', 'id_zing', 'id_nct', 'id_nhacvn', 'id_keeng', 'title', 'link_nct']);
......@@ -308,13 +311,29 @@ class TrackRepository extends BaseRepository
$query->whereNotIn('id', $crawlered);
}
if (is_array($blacklist_listen) && count($blacklist_listen) > 0) {
$query->whereNotIn('id', $blacklist_listen);
}
switch ($src) {
case 'zing': $query->whereNotNull('id_zing')->where('id_zing', '<>', ''); break;
case 'zing':
$query->whereNotNull('id_zing')
->where('id_zing', '<>', '');
if (empty($blacklist_listen)) {
// Set mặc định
$query->whereNotIn('id', [1139,3835,3664]);
}
break;
case 'nct':
$query->whereNotNull('id_nct')->where('id_nct', '<>', '');
$query->whereNotNull('id_nct')
->where('id_nct', '<>', '');
if ($is_update_composer) {
$query->whereNull('composer_list');
}
if (empty($blacklist_listen)) {
// Set mặc định
$query->whereNotIn('id', [1198,1700,1954,3223,3334,3357,3529,3688,3918,3664,3910]);
}
break;
case 'nhacvn': $query->whereNotNull('id_nhacvn')->where('id_nhacvn', '<>', ''); break;
case 'keeng': $query->whereNotNull('id_keeng')->where('id_keeng', '<>', ''); break;
......@@ -424,7 +443,7 @@ class TrackRepository extends BaseRepository
}
}
if (empty($tracks[$track['id']])) {
$tracks[$track['id']] = 'Đang Cập Nhật';
$tracks[$track['id']] = 'NIA';
}
}
......@@ -436,8 +455,16 @@ class TrackRepository extends BaseRepository
return $tracks;
}
public function getNctTrackListen($inserted)
/**
* Lấy ds blacklist để bỏ lấy lượt nghe
*
* @param $src
*
* @return array
*/
public function getBacklistListen($src)
{
<<<<<<< HEAD
// return file_get_contents('https://www.nhaccuatui.com/interaction/api/counter?listSongIds=6217224');
$tracks = [];
......@@ -484,8 +511,13 @@ class TrackRepository extends BaseRepository
} else {
return null;
}
=======
$query = CrawlerListenBlacklist::where('src', strtolower($src))
->orWhere('src', 'all_6months')
->pluck('track_id');
return $query->toArray();
>>>>>>> master
}
}
......@@ -6,8 +6,12 @@
"type": "project",
"require": {
"php": ">=5.6.4",
<<<<<<< HEAD
"fabpot/goutte": "3.2.3",
"guzzlehttp/guzzle": "6.3",
=======
"alaouy/youtube": "^2.2",
>>>>>>> master
"ixudra/curl": "^6.19",
"laravel/framework": "5.4.*",
"laravel/tinker": "~1.0",
......@@ -21,6 +25,7 @@
"yangqi/htmldom": "@dev"
},
"require-dev": {
"edupham/fillable": "dev-master",
"fzaninotto/faker": "~1.4",
"mockery/mockery": "0.9.*",
"phpunit/phpunit": "~5.7"
......
......@@ -120,7 +120,7 @@ return [
|
*/
'log' => env('APP_LOG', 'single'),
'log' => env('APP_LOG', 'daily'),
'log_level' => env('APP_LOG_LEVEL', 'debug'),
......@@ -170,6 +170,7 @@ return [
Yangqi\Htmldom\HtmldomServiceProvider::class,
Maatwebsite\Excel\ExcelServiceProvider::class,
Ixudra\Curl\CurlServiceProvider::class,
Alaouy\Youtube\YoutubeServiceProvider::class,
/*
* Application Service Providers...
......@@ -180,6 +181,8 @@ return [
App\Providers\EventServiceProvider::class,
App\Providers\RouteServiceProvider::class,
\Edupham\Fillable\EduphamFillableServiceProvider::class
],
/*
......@@ -231,6 +234,7 @@ return [
'Htmldom' => Yangqi\Htmldom\Htmldom::class,
'Excel' => Maatwebsite\Excel\Facades\Excel::class,
'Curl' => Ixudra\Curl\Facades\Curl::class,
'Youtube' => Alaouy\Youtube\Facades\Youtube::class,
],
......
......@@ -325,8 +325,8 @@ class Htmldomnode
switch ($this->nodetype)
{
case YANGQI_HDOM_TYPE_TEXT: return $this->dom->restore_noise($this->_[YANGQI_HDOM_INFO_TEXT]);
case HDOM_TYPE_COMMENT: return '';
case HDOM_TYPE_UNKNOWN: return '';
case YANGQI_HDOM_TYPE_COMMENT: return '';
case YANGQI_HDOM_TYPE_UNKNOWN: return '';
}
if (strcasecmp($this->tag, 'script')===0) return '';
if (strcasecmp($this->tag, 'style')===0) return '';
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment