add constellation crawler

app/Console/Kernel.php

@@ -2,6 +2,7 @@
 
 namespace App\Console;
 
+use App\Jobs\UpdateConstellations;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
 
@@ -24,8 +25,9 @@ class Kernel extends ConsoleKernel
      */
     protected function schedule(Schedule $schedule)
     {
-        // $schedule->command('inspire')
+        $schedule->call(function(){
-        //          ->hourly();
+            dispatch(app(UpdateConstellations::class))->onQueue('updateConstellations');
+        })->hourly();
     }
 
     /**

app/Constellation.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class Constellation extends Model
+{
+    protected $fillable = [
+        'name',
+        'all',
+        'all_desc',
+        'love',
+        'love_desc',
+        'career',
+        'career_desc',
+        'income',
+        'income_desc',
+    ];
+}

app/Http/Controllers/HomeController.php

@@ -2,7 +2,9 @@
 
 namespace App\Http\Controllers;
 
+use App\Constellation;
 use Illuminate\Http\Request;
+use DB;
 
 class HomeController extends Controller
 {
@@ -24,4 +26,26 @@ class HomeController extends Controller
     {
         return view('home');
     }
+
+    public function constellations(Request $request)
+    {
+        $dates = Constellation::groupBy('date')
+            ->orderBy('date', 'DESC')
+            ->get([
+                DB::raw('Date(created_at) as date')
+            ])->pluck('date');
+
+        $queriedDate = $request->get('date');
+        if(!$queriedDate) {
+            $queriedDate = $dates->first();
+        }
+
+        $constellations = Constellation::whereDate('created_at', $queriedDate)->get();
+
+        return view('constellations')->with([
+            'dates'          => $dates,
+            'queriedDate'    => $queriedDate,
+            'constellations' => $constellations
+        ]);
+    }
 }

app/Jobs/UpdateConstellations.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Constellation;
+use App\Services\ConstellationCrawler;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Log;
+use Illuminate\Support\Carbon;
+
+class UpdateConstellations implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $crawler = app(ConstellationCrawler::class);
+        foreach ($crawler->getConstellations() as $name => $constellation) {
+            $existedConstellation = Constellation::where('name', $name)->whereDate('created_at', Carbon::today())->first();
+            if(!$existedConstellation) {
+                $constellation = Constellation::create([
+                    'name'        => $name,
+                    'all'         => $constellation['all'],
+                    'all_desc'    => $constellation['all_desc'],
+                    'love'        => $constellation['love'],
+                    'love_desc'   => $constellation['love_desc'],
+                    'career'      => $constellation['career'],
+                    'career_desc' => $constellation['career_desc'],
+                    'income'      => $constellation['income'],
+                    'income_desc' => $constellation['income_desc'],
+                ]);
+                Log::info("{$constellation->name} updated");
+            }
+        }
+    }
+}

app/Services/ConstellationCrawler.php

@@ -0,0 +1,157 @@
+<?php
+
+namespace App\Services;
+
+use GuzzleHttp\Client;
+use Illuminate\Support\Carbon;
+
+class ConstellationCrawler
+{
+    private $baseUrl = 'http://astro.click108.com.tw/daily_1.php';
+
+    private $constellationPageIds = [
+        'Aries'       => 0,
+        'Taurus'      => 1,
+        'Gemini'      => 2,
+        'Cancer'      => 3,
+        'Leo'         => 4,
+        'Virgo'       => 5,
+        'Libra'       => 6,
+        'Scorpio'     => 7,
+        'Sagittarius' => 8,
+        'Capricorn'   => 9,
+        'Aquarius'    => 10,
+        'Pisces'      => 11,
+    ];
+
+    private function getHtml($pageId = 0)
+    {
+        $httpClient = new Client;
+        $resp = $httpClient->request('GET', $this->baseUrl . '?' . http_build_query([
+            'iAstro' => $pageId,
+            'iAcDay' => Carbon::now()->format('Y-m-d')
+        ]));
+        if($resp->getStatusCode() == 200) {
+            return $resp->getBody()->getContents();
+        } else {
+            return '';
+        }
+    }
+
+    private function removeEscapeSign($str)
+    {
+        return str_replace('\\', '', $str);
+    }
+
+    private function parseHtml($html)
+    {
+        $matches = [];
+        $entryPattern = '<div class="TODAY_CONTENT">';
+
+        $htmlPatterns = [
+            [
+                'pattern' => '<p><span class="txt_green">',
+            ],
+            [
+                'pattern' => '<\/span><\/p>',
+                'name' => 'all',
+                'type' => 'level'
+            ],
+            [
+                'pattern' => '<p>'
+            ],
+            [
+                'pattern' => '<\/p>',
+                'name' => 'all_desc',
+                'type' => 'text'
+            ],
+            [
+                'pattern' => '<p><span class="txt_pink">',
+            ],
+            [
+                'pattern' => '<\/span><\/p>',
+                'name' => 'love',
+                'type' => 'level'
+            ],
+            [
+                'pattern' => '<p>'
+            ],
+            [
+                'pattern' => '<\/p>',
+                'name' => 'love_desc',
+                'type' => 'text'
+            ],
+            [
+                'pattern' => '<p><span class="txt_blue">',
+            ],
+            [
+                'pattern' => '<\/span><\/p>',
+                'name' => 'career',
+                'type' => 'level'
+            ],
+            [
+                'pattern' => '<p>'
+            ],
+            [
+                'pattern' => '<\/p>',
+                'name' => 'career_desc',
+                'type' => 'text'
+            ],
+            [
+                'pattern' => '<p><span class="txt_orange">',
+            ],
+            [
+                'pattern' => '<\/span><\/p>',
+                'name' => 'income',
+                'type' => 'level'
+            ],
+            [
+                'pattern' => '<p>'
+            ],
+            [
+                'pattern' => '<\/p>',
+                'name' => 'income_desc',
+                'type' => 'text'
+            ],
+        ];
+
+        preg_match("/$entryPattern/", $html, $matches, PREG_OFFSET_CAPTURE);
+
+        if(!empty($matches[0])) {
+            $offset = $matches[0][1];
+            $html = trim(substr($html, $offset + strlen($entryPattern)));
+            $data = [];
+
+            foreach ($htmlPatterns as $htmlPattern) {
+                preg_match("/{$htmlPattern['pattern']}/", $html, $matches, PREG_OFFSET_CAPTURE);
+                $offset = $matches[0][1];
+                if(!empty($htmlPattern['type'])) {
+                    if($htmlPattern['type'] == 'level') {
+                        $rateText = substr($html, 0, $offset);
+                        $starMatches = [];
+                        preg_match('/★+/u', $rateText, $starMatches);
+                        $rate = empty($starMatches[0]) ? 0 : mb_strlen($starMatches[0]);
+                        $data[$htmlPattern['name']] = $rate;
+                    } else if($htmlPattern['type'] == 'text') {
+                        $data[$htmlPattern['name']] = substr($html, 0, $offset);
+                    }
+                }
+                $html = trim(substr($html, $offset + strlen($this->removeEscapeSign($htmlPattern['pattern']))));
+            }
+            return $data;
+        } else {
+            return null;
+        }
+    }
+
+    public function getConstellations()
+    {
+        $data = [];
+        foreach ($this->constellationPageIds as $name => $constellationPageId) {
+            $html = $this->getHtml($constellationPageId);
+            $data[$name] = $this->parseHtml($html);
+        }
+
+        return $data;
+    }
+}

composer.json

@@ -11,8 +11,10 @@
         "php": "^7.2",
         "fideloper/proxy": "^4.0",
         "google/apiclient": "^2.4",
+        "guzzlehttp/guzzle": "^6.5",
         "laravel/framework": "^6.2",
-        "laravel/tinker": "^2.0"
+        "laravel/tinker": "^2.0",
+        "predis/predis": "^1.1"
     },
     "require-dev": {
         "facade/ignition": "^1.4",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "content-hash": "f8c1c5bbfc814eceb05f09efe571e699",
+    "content-hash": "4d733788f8dc0c1f6d18ade50ac5fc42",
     "packages": [
         {
             "name": "dnoegel/php-xdg-base-dir",
@@ -1631,6 +1631,56 @@
             ],
             "time": "2020-02-25T04:16:50+00:00"
         },
+        {
+            "name": "predis/predis",
+            "version": "v1.1.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/nrk/predis.git",
+                "reference": "f0210e38881631afeafb56ab43405a92cafd9fd1"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/nrk/predis/zipball/f0210e38881631afeafb56ab43405a92cafd9fd1",
+                "reference": "f0210e38881631afeafb56ab43405a92cafd9fd1",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.9"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "~4.8"
+            },
+            "suggest": {
+                "ext-curl": "Allows access to Webdis when paired with phpiredis",
+                "ext-phpiredis": "Allows faster serialization and deserialization of the Redis protocol"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Predis\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Daniele Alessandri",
+                    "email": "suppakilla@gmail.com",
+                    "homepage": "http://clorophilla.net"
+                }
+            ],
+            "description": "Flexible and feature-complete Redis client for PHP and HHVM",
+            "homepage": "http://github.com/nrk/predis",
+            "keywords": [
+                "nosql",
+                "predis",
+                "redis"
+            ],
+            "time": "2016-06-16T16:22:20+00:00"
+        },
         {
             "name": "psr/cache",
             "version": "1.0.1",

database/migrations/2020_03_04_002130_create_constellations_table.php

@@ -0,0 +1,40 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateConstellationsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('constellations', function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->string('name', 11);
+            $table->unsignedTinyInteger('all');
+            $table->text('all_desc');
+            $table->unsignedTinyInteger('love');
+            $table->text('love_desc');
+            $table->unsignedTinyInteger('career');
+            $table->text('career_desc');
+            $table->unsignedTinyInteger('income');
+            $table->text('income_desc');
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('constellations');
+    }
+}

resources/views/constellations.blade.php

@@ -0,0 +1,55 @@
+@extends('layouts.app')
+
+@section('content')
+    <div class="container">
+        <div class="row">
+            <div class="col">
+                <form method="GET">
+                    <div class="form-group">
+                        <select name="date">
+                            @foreach($dates as $date)
+                                <option value="{{ $date }}" {{ $queriedDate == $date ? 'selected="selected"' : '' }}>{{ $date }}</option>
+                            @endforeach
+                        </select>
+                        <button class="btn btn-success">Query</button>
+                    </div>
+                </form>
+            </div>
+        </div>
+        <div class="row">
+            <div class="col">
+                <table class="table table-striped">
+                    <thead>
+                    <tr>
+                        <th>名稱</th>
+                        <th>整體運勢</th>
+                        <th>整理運勢說明</th>
+                        <th>愛情運勢</th>
+                        <th>愛情運勢說明</th>
+                        <th>事業運勢</th>
+                        <th>事業運勢說明</th>
+                        <th>財運運勢</th>
+                        <th>財運運勢說明</th>
+                    </tr>
+                    </thead>
+                    <tbody>
+                    @foreach($constellations as $constellation)
+                        <tr>
+                            <td>{{ $constellation->name }}</td>
+                            <td>{{ $constellation->all }}</td>
+                            <td>{{ $constellation->all_desc }}</td>
+                            <td>{{ $constellation->love }}</td>
+                            <td>{{ $constellation->love_desc }}</td>
+                            <td>{{ $constellation->career }}</td>
+                            <td>{{ $constellation->career_desc }}</td>
+                            <td>{{ $constellation->income }}</td>
+                            <td>{{ $constellation->income_desc }}</td>
+                        </tr>
+                    @endforeach
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+
+@endsection

resources/views/layouts/app.blade.php

@@ -37,7 +37,9 @@
                 <div class="collapse navbar-collapse" id="navbarSupportedContent">
                     <!-- Left Side Of Navbar -->
                     <ul class="navbar-nav mr-auto">
-
+                        <li class="nav-item">
+                            <a href="{{ route('constellations') }}" class="nav-link">Constellations</a>
+                        </li>
                     </ul>
 
                     <!-- Right Side Of Navbar -->

routes/web.php

@@ -16,3 +16,4 @@ Auth::routes();
 Route::get('/', 'HomeController@index')->name('index');
 
 Route::post('/google-login', 'Auth\LoginController@googleLogin');
+Route::get('/constellations', 'HomeController@constellations')->name('constellations');