public-projects-laravel/review
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add constellation crawler

Browse Source
This commit is contained in:
kroutony 2020-03-04 11:26:23 +08:00
parent 7593b83632
commit 6cee1f3f0d
16 changed files with 3084 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Console/Kernel.php
View File

@ -2,6 +2,7 @@
namespace App\Console;
use App\Jobs\UpdateConstellations;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@ -24,8 +25,9 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
// $schedule->command('inspire')
// ->hourly();
$schedule->call(function(){
dispatch(app(UpdateConstellations::class))->onQueue('updateConstellations');
})->hourly();
}
/**

20
app/Constellation.php Normal file
View File

@ -0,0 +1,20 @@
<?php
namespace App;
use Illuminate\Database\Eloquent\Model;
class Constellation extends Model
{
protected $fillable = [
'name',
'all',
'all_desc',
'love',
'love_desc',
'career',
'career_desc',
'income',
'income_desc',
];
}

24
app/Http/Controllers/HomeController.php
View File

@ -2,7 +2,9 @@
namespace App\Http\Controllers;
use App\Constellation;
use Illuminate\Http\Request;
use DB;
class HomeController extends Controller
{
@ -24,4 +26,26 @@ class HomeController extends Controller
{
return view('home');
}
public function constellations(Request $request)
{
$dates = Constellation::groupBy('date')
->orderBy('date', 'DESC')
->get([
DB::raw('Date(created_at) as date')
])->pluck('date');
$queriedDate = $request->get('date');
if(!$queriedDate) {
$queriedDate = $dates->first();
}
$constellations = Constellation::whereDate('created_at', $queriedDate)->get();
return view('constellations')->with([
'dates' => $dates,
'queriedDate' => $queriedDate,
'constellations' => $constellations
]);
}
}

55
app/Jobs/UpdateConstellations.php Normal file
View File

@ -0,0 +1,55 @@
<?php
namespace App\Jobs;
use App\Constellation;
use App\Services\ConstellationCrawler;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Carbon;
class UpdateConstellations implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct()
{
//
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$crawler = app(ConstellationCrawler::class);
foreach ($crawler->getConstellations() as $name => $constellation) {
$existedConstellation = Constellation::where('name', $name)->whereDate('created_at', Carbon::today())->first();
if(!$existedConstellation) {
$constellation = Constellation::create([
'name' => $name,
'all' => $constellation['all'],
'all_desc' => $constellation['all_desc'],
'love' => $constellation['love'],
'love_desc' => $constellation['love_desc'],
'career' => $constellation['career'],
'career_desc' => $constellation['career_desc'],
'income' => $constellation['income'],
'income_desc' => $constellation['income_desc'],
]);
Log::info("{$constellation->name} updated");
}
}
}
}

157
app/Services/ConstellationCrawler.php Normal file
View File

@ -0,0 +1,157 @@
<?php
namespace App\Services;
use GuzzleHttp\Client;
use Illuminate\Support\Carbon;
class ConstellationCrawler
{
private $baseUrl = 'http://astro.click108.com.tw/daily_1.php';
private $constellationPageIds = [
'Aries' => 0,
'Taurus' => 1,
'Gemini' => 2,
'Cancer' => 3,
'Leo' => 4,
'Virgo' => 5,
'Libra' => 6,
'Scorpio' => 7,
'Sagittarius' => 8,
'Capricorn' => 9,
'Aquarius' => 10,
'Pisces' => 11,
];
private function getHtml($pageId = 0)
{
$httpClient = new Client;
$resp = $httpClient->request('GET', $this->baseUrl . '?' . http_build_query([
'iAstro' => $pageId,
'iAcDay' => Carbon::now()->format('Y-m-d')
]));
if($resp->getStatusCode() == 200) {
return $resp->getBody()->getContents();
} else {
return '';
}
}
private function removeEscapeSign($str)
{
return str_replace('\\', '', $str);
}
private function parseHtml($html)
{
$matches = [];
$entryPattern = '<div class="TODAY_CONTENT">';
$htmlPatterns = [
[
'pattern' => '<p><span class="txt_green">',
],
[
'pattern' => '<\/span><\/p>',
'name' => 'all',
'type' => 'level'
],
[
'pattern' => '<p>'
],
[
'pattern' => '<\/p>',
'name' => 'all_desc',
'type' => 'text'
],
[
'pattern' => '<p><span class="txt_pink">',
],
[
'pattern' => '<\/span><\/p>',
'name' => 'love',
'type' => 'level'
],
[
'pattern' => '<p>'
],
[
'pattern' => '<\/p>',
'name' => 'love_desc',
'type' => 'text'
],
[
'pattern' => '<p><span class="txt_blue">',
],
[
'pattern' => '<\/span><\/p>',
'name' => 'career',
'type' => 'level'
],
[
'pattern' => '<p>'
],
[
'pattern' => '<\/p>',
'name' => 'career_desc',
'type' => 'text'
],
[
'pattern' => '<p><span class="txt_orange">',
],
[
'pattern' => '<\/span><\/p>',
'name' => 'income',
'type' => 'level'
],
[
'pattern' => '<p>'
],
[
'pattern' => '<\/p>',
'name' => 'income_desc',
'type' => 'text'
],
];
preg_match("/$entryPattern/", $html, $matches, PREG_OFFSET_CAPTURE);
if(!empty($matches[0])) {
$offset = $matches[0][1];
$html = trim(substr($html, $offset + strlen($entryPattern)));
$data = [];
foreach ($htmlPatterns as $htmlPattern) {
preg_match("/{$htmlPattern['pattern']}/", $html, $matches, PREG_OFFSET_CAPTURE);
$offset = $matches[0][1];
if(!empty($htmlPattern['type'])) {
if($htmlPattern['type'] == 'level') {
$rateText = substr($html, 0, $offset);
$starMatches = [];
preg_match('/★+/u', $rateText, $starMatches);
$rate = empty($starMatches[0]) ? 0 : mb_strlen($starMatches[0]);
$data[$htmlPattern['name']] = $rate;
} else if($htmlPattern['type'] == 'text') {
$data[$htmlPattern['name']] = substr($html, 0, $offset);
}
}
$html = trim(substr($html, $offset + strlen($this->removeEscapeSign($htmlPattern['pattern']))));
}
return $data;
} else {
return null;
}
}
public function getConstellations()
{
$data = [];
foreach ($this->constellationPageIds as $name => $constellationPageId) {
$html = $this->getHtml($constellationPageId);
$data[$name] = $this->parseHtml($html);
}
return $data;
}
}

4
composer.json
View File

@ -11,8 +11,10 @@
"php": "^7.2",
"fideloper/proxy": "^4.0",
"google/apiclient": "^2.4",
"guzzlehttp/guzzle": "^6.5",
"laravel/framework": "^6.2",
"laravel/tinker": "^2.0"
"laravel/tinker": "^2.0",
"predis/predis": "^1.1"
},
"require-dev": {
"facade/ignition": "^1.4",

52
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "f8c1c5bbfc814eceb05f09efe571e699",
"content-hash": "4d733788f8dc0c1f6d18ade50ac5fc42",
"packages": [
{
"name": "dnoegel/php-xdg-base-dir",
@ -1631,6 +1631,56 @@
],
"time": "2020-02-25T04:16:50+00:00"
},
{
"name": "predis/predis",
"version": "v1.1.1",
"source": {
"type": "git",
"url": "https://github.com/nrk/predis.git",
"reference": "f0210e38881631afeafb56ab43405a92cafd9fd1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/nrk/predis/zipball/f0210e38881631afeafb56ab43405a92cafd9fd1",
"reference": "f0210e38881631afeafb56ab43405a92cafd9fd1",
"shasum": ""
},
"require": {
"php": ">=5.3.9"
},
"require-dev": {
"phpunit/phpunit": "~4.8"
},
"suggest": {
"ext-curl": "Allows access to Webdis when paired with phpiredis",
"ext-phpiredis": "Allows faster serialization and deserialization of the Redis protocol"
},
"type": "library",
"autoload": {
"psr-4": {
"Predis\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Daniele Alessandri",
"email": "suppakilla@gmail.com",
"homepage": "http://clorophilla.net"
}
],
"description": "Flexible and feature-complete Redis client for PHP and HHVM",
"homepage": "http://github.com/nrk/predis",
"keywords": [
"nosql",
"predis",
"redis"
],
"time": "2016-06-16T16:22:20+00:00"
},
{
"name": "psr/cache",
"version": "1.0.1",

40
database/migrations/2020_03_04_002130_create_constellations_table.php Normal file
View File

@ -0,0 +1,40 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateConstellationsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('constellations', function (Blueprint $table) {
$table->bigIncrements('id');
$table->string('name', 11);
$table->unsignedTinyInteger('all');
$table->text('all_desc');
$table->unsignedTinyInteger('love');
$table->text('love_desc');
$table->unsignedTinyInteger('career');
$table->text('career_desc');
$table->unsignedTinyInteger('income');
$table->text('income_desc');
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('constellations');
}
}

BIN
public/fonts/vendor/font-awesome/fontawesome-webfont.eot vendored Normal file
View File

Binary file not shown.

2671
public/fonts/vendor/font-awesome/fontawesome-webfont.svg vendored Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 434 KiB

BIN
public/fonts/vendor/font-awesome/fontawesome-webfont.ttf vendored Normal file
View File

Binary file not shown.

BIN
public/fonts/vendor/font-awesome/fontawesome-webfont.woff vendored Normal file
View File

Binary file not shown.

BIN
public/fonts/vendor/font-awesome/fontawesome-webfont.woff2 vendored Normal file
View File

Binary file not shown.

55
resources/views/constellations.blade.php Normal file
View File

@ -0,0 +1,55 @@
@extends('layouts.app')
@section('content')
<div class="container">
<div class="row">
<div class="col">
<form method="GET">
<div class="form-group">
<select name="date">
@foreach($dates as $date)
<option value="{{ $date }}" {{ $queriedDate == $date ? 'selected="selected"' : '' }}>{{ $date }}</option>
@endforeach
</select>
<button class="btn btn-success">Query</button>
</div>
</form>
</div>
</div>
<div class="row">
<div class="col">
<table class="table table-striped">
<thead>
<tr>
<th>名稱</th>
<th>整體運勢</th>
<th>整理運勢說明</th>
<th>愛情運勢</th>
<th>愛情運勢說明</th>
<th>事業運勢</th>
<th>事業運勢說明</th>
<th>財運運勢</th>
<th>財運運勢說明</th>
</tr>
</thead>
<tbody>
@foreach($constellations as $constellation)
<tr>
<td>{{ $constellation->name }}</td>
<td>{{ $constellation->all }}</td>
<td>{{ $constellation->all_desc }}</td>
<td>{{ $constellation->love }}</td>
<td>{{ $constellation->love_desc }}</td>
<td>{{ $constellation->career }}</td>
<td>{{ $constellation->career_desc }}</td>
<td>{{ $constellation->income }}</td>
<td>{{ $constellation->income_desc }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
@endsection

4
resources/views/layouts/app.blade.php
View File

@ -37,7 +37,9 @@
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<!-- Left Side Of Navbar -->
<ul class="navbar-nav mr-auto">
<li class="nav-item">
<a href="{{ route('constellations') }}" class="nav-link">Constellations</a>
</li>
</ul>
<!-- Right Side Of Navbar -->

1
routes/web.php
View File

@ -16,3 +16,4 @@ Auth::routes();
Route::get('/', 'HomeController@index')->name('index');
Route::post('/google-login', 'Auth\LoginController@googleLogin');
Route::get('/constellations', 'HomeController@constellations')->name('constellations');